Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

only create ghost records when it is not an incremental run #250

Merged
merged 3 commits into from
Sep 5, 2024
Merged

only create ghost records when it is not an incremental run #250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d0139b
only create ghost records when it is not an incremental run
tkiehn Sep 4, 2024
e18090c
fix additional endif
tkiehn Sep 4, 2024
34098fd
Trigger Pipeline
tkiehn Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions macros/staging/bigquery/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -597,7 +597,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{%- if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL

SELECT
Expand Down
4 changes: 2 additions & 2 deletions macros/staging/exasol/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (
SELECT
Expand Down Expand Up @@ -580,7 +580,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{%- if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL
SELECT

Expand Down
4 changes: 2 additions & 2 deletions macros/staging/postgres/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{%- if enable_ghost_records -%}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -597,7 +597,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL

SELECT
Expand Down
4 changes: 2 additions & 2 deletions macros/staging/redshift/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{%- if enable_ghost_records -%}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -606,7 +606,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{%- if enable_ghost_records %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL

SELECT
Expand Down
8 changes: 2 additions & 6 deletions macros/staging/snowflake/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,8 +389,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -535,7 +534,6 @@ ghost_records AS (
SELECT * FROM error_values
),
{%- endif %}
{%- endif %}

{%- if not include_source_columns -%}
{% set source_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=source_columns_to_select, exclude_columns_list=derived_column_names) %}
Expand All @@ -551,16 +549,14 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL

SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{% endif %}
{% endif %}
)

Expand Down
8 changes: 2 additions & 6 deletions macros/staging/synapse/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,8 +439,7 @@ hashed_columns AS (
{% set processed_hash_columns = datavault4dbt.process_hash_column_excludes(hashed_columns) -%}
{%- endif -%}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{%- if enable_ghost_records and not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -585,7 +584,6 @@ ghost_records AS (
SELECT * FROM error_values
),
{%- endif %}
{%- endif %}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
Expand All @@ -600,16 +598,14 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{%- if enable_ghost_records and not is_incremental() %}
UNION ALL

SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{% endif %}
{% endif %}
)

Expand Down