Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add possibility to switch of ghost records in the staging macro #192

Merged
merged 6 commits into from
Sep 3, 2024
Merged

Add possibility to switch of ghost records in the staging macro #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f51f4cc
add enable_ghost_records_parameter to stages, makes it possible to di…
tkiehn Jun 28, 2024
db98800
fix wrong endif
tkiehn Jun 28, 2024
46b8c8e
Update README.md
tkirschke Sep 2, 2024
72982ef
Trigger Pipeline
tkiehn Sep 3, 2024
2640394
Merge branch 'main' into 99-ghost-records-may-not-be-switched-off
tkiehn Sep 3, 2024
3d858c0
fix whitespace control
tkiehn Sep 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions macros/staging/bigquery/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -147,7 +148,7 @@

{%- set source_columns_to_select = only_include_from_source -%}

{%- endif-%}
{%- endif -%}

{%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%}
{%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
Expand Down Expand Up @@ -437,6 +438,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -580,6 +582,7 @@ ghost_records AS (
UNION ALL
SELECT * FROM error_values
),
{%- endif -%}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
Expand All @@ -594,13 +597,15 @@ columns_to_select AS (

FROM {{ last_cte }}

{%- if enable_ghost_records %}
UNION ALL

SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{%- endif %}
)

SELECT * FROM columns_to_select
Expand Down
12 changes: 10 additions & 2 deletions macros/staging/exasol/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -142,7 +143,7 @@

{%- set source_columns_to_select = only_include_from_source -%}

{%- endif-%}
{%- endif -%}

{%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%}
{%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
Expand Down Expand Up @@ -425,6 +426,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (
SELECT
Expand Down Expand Up @@ -563,9 +565,12 @@ ghost_records AS (
UNION ALL
SELECT * FROM error_values
),
{%- endif -%}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
{%- endif -%}

{# Combining the two ghost records with the regular data #}
columns_to_select AS (

Expand All @@ -574,12 +579,15 @@ columns_to_select AS (
{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM {{ last_cte }}

{%- if enable_ghost_records %}
UNION ALL
SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{%- endif -%}
)

SELECT * FROM columns_to_select
Expand Down
9 changes: 7 additions & 2 deletions macros/staging/postgres/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -148,7 +149,7 @@

{%- set source_columns_to_select = only_include_from_source -%}

{%- endif-%}
{%- endif -%}

{%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%}
{%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
Expand Down Expand Up @@ -437,6 +438,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{%- if enable_ghost_records -%}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -580,6 +582,7 @@ ghost_records AS (
UNION ALL
SELECT * FROM error_values
),
{%- endif -%}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
Expand All @@ -594,13 +597,15 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
UNION ALL

SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{% endif %}
)

SELECT * FROM columns_to_select
Expand Down
9 changes: 7 additions & 2 deletions macros/staging/redshift/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -147,7 +148,7 @@

{%- set source_columns_to_select = only_include_from_source -%}

{%- endif-%}
{%- endif -%}

{%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%}
{%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
Expand Down Expand Up @@ -428,6 +429,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{%- if enable_ghost_records -%}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (

Expand Down Expand Up @@ -571,6 +573,7 @@ ghost_records AS (
UNION ALL
SELECT * FROM error_values
),
{%- endif -%}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
Expand All @@ -585,13 +588,15 @@ columns_to_select AS (

FROM {{ last_cte }}

{%- if enable_ghost_records %}
UNION ALL

SELECT

{{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }}

FROM ghost_records
{%- endif -%}
)

SELECT * FROM columns_to_select
Expand Down
9 changes: 7 additions & 2 deletions macros/staging/snowflake/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -151,7 +152,7 @@
{%- set source_columns_to_select = only_include_from_source -%}
{{ log('source_columns_to_select when include_source_columns=false: '~ source_columns_to_select, false) }}

{%- endif-%}
{%- endif -%}

{%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%}
{%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
Expand Down Expand Up @@ -388,6 +389,7 @@ hashed_columns AS (
{%- endif -%}
{%- endif -%}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (
Expand Down Expand Up @@ -533,6 +535,7 @@ ghost_records AS (
SELECT * FROM error_values
),
{%- endif %}
{%- endif %}

{%- if not include_source_columns -%}
{% set source_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=source_columns_to_select, exclude_columns_list=derived_column_names) %}
Expand All @@ -548,6 +551,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
{% if not is_incremental() %}
UNION ALL

Expand All @@ -557,6 +561,7 @@ columns_to_select AS (

FROM ghost_records
{% endif %}
{% endif %}
)

SELECT * FROM columns_to_select
Expand Down
12 changes: 10 additions & 2 deletions macros/staging/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,24 @@
{'multi_active_key': ['phonetype', 'company'], This source data comes with two multi-active keys. The combination of those two, the main_hashkey and ldts is unique
'main_hashkey_column': 'hk_contact_h'} inside the source system.

enable_ghost_records::boolean If set to true, the stage will be created with ghost records. By default, ghost records are enabled. Optional Parameter

#}



{%- macro stage(ldts, rsrc, source_model, include_source_columns=true, hashed_columns=none, derived_columns=none, sequence=none, prejoined_columns=none, missing_columns=none, multi_active_config=none) -%}
{%- macro stage(ldts, rsrc, source_model, include_source_columns=true, hashed_columns=none, derived_columns=none, sequence=none, prejoined_columns=none, missing_columns=none, multi_active_config=none, enable_ghost_records=true) -%}

{# If include_source_columns is passed but its empty then it is set with the default value (true) #}
{%- if include_source_columns is none or include_source_columns == "" -%}
{%- set include_source_columns = true -%}
{%- endif -%}

{# If enable_ghost_records is passed but its empty then it is set with the default value (true) #}
{%- if enable_ghost_records is none or enable_ghost_records == "" -%}
{%- set enable_ghost_records = true -%}
{%- endif -%}

{# If ldts is empty replace it with the current timestamp #}
{%- if datavault4dbt.is_nothing(ldts) -%}
{%- set ldts = datavault4dbt.current_timestamp() -%}
Expand All @@ -123,6 +130,7 @@
sequence=sequence,
prejoined_columns=prejoined_columns,
missing_columns=missing_columns,
multi_active_config=multi_active_config) -}}
multi_active_config=multi_active_config,
enable_ghost_records=enable_ghost_records) -}}

{%- endmacro -%}
9 changes: 7 additions & 2 deletions macros/staging/synapse/stage.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
sequence,
prejoined_columns,
missing_columns,
multi_active_config) -%}
multi_active_config,
enable_ghost_records) -%}

{% if (source_model is none) and execute %}

Expand Down Expand Up @@ -138,7 +139,7 @@
{%- endif -%}
{%- set source_columns_to_select = only_include_from_source -%}

{%- endif-%}
{%- endif -%}

{%- if not var('datavault4dbt.include_derived_column_input_columns', true) -%}
{%- set exclude_column_names = exclude_column_names + derived_input_columns -%}
Expand Down Expand Up @@ -438,6 +439,7 @@ hashed_columns AS (
{% set processed_hash_columns = datavault4dbt.process_hash_column_excludes(hashed_columns) -%}
{%- endif -%}

{% if enable_ghost_records %}
{% if not is_incremental() %}
{# Creating Ghost Record for unknown case, based on datatype #}
unknown_values AS (
Expand Down Expand Up @@ -583,6 +585,7 @@ ghost_records AS (
SELECT * FROM error_values
),
{%- endif %}
{%- endif %}

{%- if not include_source_columns -%}
{% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %}
Expand All @@ -597,6 +600,7 @@ columns_to_select AS (

FROM {{ last_cte }}

{% if enable_ghost_records %}
{% if not is_incremental() %}
UNION ALL

Expand All @@ -606,6 +610,7 @@ columns_to_select AS (

FROM ghost_records
{% endif %}
{% endif %}
)

SELECT * FROM columns_to_select
Expand Down