diff --git a/macros/staging/bigquery/stage.sql b/macros/staging/bigquery/stage.sql index 80ff3c38..e8eb5789 100644 --- a/macros/staging/bigquery/stage.sql +++ b/macros/staging/bigquery/stage.sql @@ -9,7 +9,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -147,7 +148,7 @@ {%- set source_columns_to_select = only_include_from_source -%} -{%- endif-%} +{%- endif -%} {%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%} {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%} @@ -437,6 +438,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} +{% if enable_ghost_records %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -580,6 +582,7 @@ ghost_records AS ( UNION ALL SELECT * FROM error_values ), +{%- endif -%} {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} @@ -594,13 +597,15 @@ columns_to_select AS ( FROM {{ last_cte }} +{%- if enable_ghost_records %} UNION ALL - + SELECT {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records +{%- endif %} ) SELECT * FROM columns_to_select diff --git a/macros/staging/exasol/stage.sql b/macros/staging/exasol/stage.sql index a7acf53a..3935599d 100644 --- a/macros/staging/exasol/stage.sql +++ b/macros/staging/exasol/stage.sql @@ -7,7 +7,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -142,7 +143,7 @@ {%- set source_columns_to_select = only_include_from_source -%} -{%- endif-%} +{%- endif -%} {%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%} {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%} @@ -425,6 +426,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} +{% if enable_ghost_records %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( SELECT @@ -563,9 +565,12 @@ ghost_records AS ( UNION ALL SELECT * FROM error_values ), +{%- endif -%} + {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} {%- endif -%} + {# Combining the two ghost records with the regular data #} columns_to_select AS ( @@ -574,12 +579,15 @@ columns_to_select AS ( {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM {{ last_cte }} + +{%- if enable_ghost_records %} UNION ALL SELECT {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records +{%- endif -%} ) SELECT * FROM columns_to_select diff --git a/macros/staging/postgres/stage.sql b/macros/staging/postgres/stage.sql index a6126412..d5a3d0ad 100644 --- a/macros/staging/postgres/stage.sql +++ b/macros/staging/postgres/stage.sql @@ -9,7 +9,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -148,7 +149,7 @@ {%- set source_columns_to_select = only_include_from_source -%} -{%- endif-%} +{%- endif -%} {%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%} {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%} @@ -437,6 +438,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} +{%- if enable_ghost_records -%} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -580,6 +582,7 @@ ghost_records AS ( UNION ALL SELECT * FROM error_values ), +{%- endif -%} {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} @@ -594,6 +597,7 @@ columns_to_select AS ( FROM {{ last_cte }} +{% if enable_ghost_records %} UNION ALL SELECT @@ -601,6 +605,7 @@ columns_to_select AS ( {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records +{% endif %} ) SELECT * FROM columns_to_select diff --git a/macros/staging/redshift/stage.sql b/macros/staging/redshift/stage.sql index b3b5c56f..97ed0304 100644 --- a/macros/staging/redshift/stage.sql +++ b/macros/staging/redshift/stage.sql @@ -9,7 +9,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -147,7 +148,7 @@ {%- set source_columns_to_select = only_include_from_source -%} -{%- endif-%} +{%- endif -%} {%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%} {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%} @@ -428,6 +429,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} +{%- if enable_ghost_records -%} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -571,6 +573,7 @@ ghost_records AS ( UNION ALL SELECT * FROM error_values ), +{%- endif -%} {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} @@ -585,6 +588,7 @@ columns_to_select AS ( FROM {{ last_cte }} +{%- if enable_ghost_records %} UNION ALL SELECT @@ -592,6 +596,7 @@ columns_to_select AS ( {{ datavault4dbt.print_list(datavault4dbt.escape_column_names(final_columns_to_select)) }} FROM ghost_records +{%- endif -%} ) SELECT * FROM columns_to_select diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql index 38e5c92c..2a04f34e 100644 --- a/macros/staging/snowflake/stage.sql +++ b/macros/staging/snowflake/stage.sql @@ -9,7 +9,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -151,7 +152,7 @@ {%- set source_columns_to_select = only_include_from_source -%} {{ log('source_columns_to_select when include_source_columns=false: '~ source_columns_to_select, false) }} -{%- endif-%} +{%- endif -%} {%- set final_columns_to_select = final_columns_to_select + source_columns_to_select -%} {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%} @@ -388,6 +389,7 @@ hashed_columns AS ( {%- endif -%} {%- endif -%} +{% if enable_ghost_records %} {% if not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -533,6 +535,7 @@ ghost_records AS ( SELECT * FROM error_values ), {%- endif %} +{%- endif %} {%- if not include_source_columns -%} {% set source_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=source_columns_to_select, exclude_columns_list=derived_column_names) %} @@ -548,6 +551,7 @@ columns_to_select AS ( FROM {{ last_cte }} +{% if enable_ghost_records %} {% if not is_incremental() %} UNION ALL @@ -557,6 +561,7 @@ columns_to_select AS ( FROM ghost_records {% endif %} +{% endif %} ) SELECT * FROM columns_to_select diff --git a/macros/staging/stage.sql b/macros/staging/stage.sql index 5d7185df..76b17ed0 100644 --- a/macros/staging/stage.sql +++ b/macros/staging/stage.sql @@ -98,17 +98,24 @@ {'multi_active_key': ['phonetype', 'company'], This source data comes with two multi-active keys. The combination of those two, the main_hashkey and ldts is unique 'main_hashkey_column': 'hk_contact_h'} inside the source system. + enable_ghost_records::boolean If set to true, the stage will be created with ghost records. By default, ghost records are enabled. Optional Parameter + #} - {%- macro stage(ldts, rsrc, source_model, include_source_columns=true, hashed_columns=none, derived_columns=none, sequence=none, prejoined_columns=none, missing_columns=none, multi_active_config=none) -%} + {%- macro stage(ldts, rsrc, source_model, include_source_columns=true, hashed_columns=none, derived_columns=none, sequence=none, prejoined_columns=none, missing_columns=none, multi_active_config=none, enable_ghost_records=true) -%} {# If include_source_columns is passed but its empty then it is set with the default value (true) #} {%- if include_source_columns is none or include_source_columns == "" -%} {%- set include_source_columns = true -%} {%- endif -%} + {# If enable_ghost_records is passed but its empty then it is set with the default value (true) #} + {%- if enable_ghost_records is none or enable_ghost_records == "" -%} + {%- set enable_ghost_records = true -%} + {%- endif -%} + {# If ldts is empty replace it with the current timestamp #} {%- if datavault4dbt.is_nothing(ldts) -%} {%- set ldts = datavault4dbt.current_timestamp() -%} @@ -123,6 +130,7 @@ sequence=sequence, prejoined_columns=prejoined_columns, missing_columns=missing_columns, - multi_active_config=multi_active_config) -}} + multi_active_config=multi_active_config, + enable_ghost_records=enable_ghost_records) -}} {%- endmacro -%} diff --git a/macros/staging/synapse/stage.sql b/macros/staging/synapse/stage.sql index 4cb39326..027f604b 100644 --- a/macros/staging/synapse/stage.sql +++ b/macros/staging/synapse/stage.sql @@ -7,7 +7,8 @@ sequence, prejoined_columns, missing_columns, - multi_active_config) -%} + multi_active_config, + enable_ghost_records) -%} {% if (source_model is none) and execute %} @@ -138,7 +139,7 @@ {%- endif -%} {%- set source_columns_to_select = only_include_from_source -%} -{%- endif-%} +{%- endif -%} {%- if not var('datavault4dbt.include_derived_column_input_columns', true) -%} {%- set exclude_column_names = exclude_column_names + derived_input_columns -%} @@ -438,6 +439,7 @@ hashed_columns AS ( {% set processed_hash_columns = datavault4dbt.process_hash_column_excludes(hashed_columns) -%} {%- endif -%} +{% if enable_ghost_records %} {% if not is_incremental() %} {# Creating Ghost Record for unknown case, based on datatype #} unknown_values AS ( @@ -583,6 +585,7 @@ ghost_records AS ( SELECT * FROM error_values ), {%- endif %} +{%- endif %} {%- if not include_source_columns -%} {% set final_columns_to_select = datavault4dbt.process_columns_to_select(columns_list=final_columns_to_select, exclude_columns_list=source_columns_to_select) %} @@ -597,6 +600,7 @@ columns_to_select AS ( FROM {{ last_cte }} +{% if enable_ghost_records %} {% if not is_incremental() %} UNION ALL @@ -606,6 +610,7 @@ columns_to_select AS ( FROM ghost_records {% endif %} +{% endif %} ) SELECT * FROM columns_to_select