Merge pull request #274 from ScalefreeCOM/140-Optionality-for-NHLink-…

…foreign-hashkeys 140 optionality for nh link foreign hashkeys
ScalefreeCOM · Oct 30, 2024 · 9ecd6d0 · 9ecd6d0
2 parents a80a51b + 4fae523
commit 9ecd6d0
Show file tree

Hide file tree

Showing 10 changed files with 119 additions and 64 deletions.
diff --git a/macros/tables/bigquery/nh_link.sql b/macros/tables/bigquery/nh_link.sql
@@ -1,17 +1,10 @@
 {%- macro default__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
 
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
 
-
 {# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
 {# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -21,10 +14,21 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +255,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{%- endif -%}
+
 {%- endmacro -%}
diff --git a/macros/tables/databricks/nh_link.sql b/macros/tables/databricks/nh_link.sql
@@ -1,16 +1,10 @@
 {%- macro databricks__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
 
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
-
+{{ log('source_models: '~source_models, false) }}
 
 {# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
@@ -21,10 +15,20 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +255,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{%- endif -%}
+
 {%- endmacro -%}
diff --git a/macros/tables/exasol/nh_link.sql b/macros/tables/exasol/nh_link.sql
@@ -5,7 +5,6 @@
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
 
-
 {# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
 {# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -15,10 +14,21 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -245,4 +255,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{% endif %}
+
 {%- endmacro -%}
diff --git a/macros/tables/fabric/nh_link.sql b/macros/tables/fabric/nh_link.sql
@@ -1,13 +1,5 @@
 {%- macro fabric__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
 
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
-
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
-
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set beginning_of_all_times = datavault4dbt.beginning_of_all_times() -%}
@@ -24,10 +16,20 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {%- set final_columns_to_select = datavault4dbt.escape_column_names(final_columns_to_select) -%}
@@ -273,5 +275,6 @@ records_to_insert AS (
 )
 
 SELECT * FROM records_to_insert
+{% endif %}
 
 {%- endmacro -%}
diff --git a/macros/tables/nh_link.sql b/macros/tables/nh_link.sql
@@ -82,8 +82,7 @@
 
 #}
 
-{%- macro nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts=none, src_rsrc=none, disable_hwm=false, source_is_single_batch=false) -%}
-
+{%- macro nh_link(link_hashkey, payload, source_models, foreign_hashkeys=none, src_ldts=none, src_rsrc=none, disable_hwm=false, source_is_single_batch=false) -%}
     {# Applying the default aliases as stored inside the global variables, if src_ldts and src_rsrc are not set. #}
 
     {%- set src_ldts = datavault4dbt.replace_standard(src_ldts, 'datavault4dbt.ldts_alias', 'ldts') -%}

diff --git a/macros/tables/oracle/nh_link.sql b/macros/tables/oracle/nh_link.sql
@@ -1,16 +1,10 @@
 {%- macro oracle__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
-
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
+{{ log('source_models: '~source_models, false) }}
 
 {# If no specific link_hk and fk_columns are defined for each source, we apply the values set in the link_hashkey and foreign_hashkeys variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
@@ -21,10 +15,21 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -257,4 +262,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{%- endif -%}
+
 {%- endmacro -%}
diff --git a/macros/tables/postgres/nh_link.sql b/macros/tables/postgres/nh_link.sql
@@ -1,16 +1,10 @@
 {%- macro postgres__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
 
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
-
+{{ log('source_models: '~source_models, false) }}
 
 {# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
@@ -21,10 +15,21 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -255,4 +260,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{%- endif -%}
+
 {%- endmacro -%}
diff --git a/macros/tables/redshift/nh_link.sql b/macros/tables/redshift/nh_link.sql
@@ -1,17 +1,10 @@
 {%- macro redshift__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
 
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
 
-
 {# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
 {# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -21,10 +14,21 @@
 
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -255,4 +259,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{% endif %}
+
 {%- endmacro -%}
diff --git a/macros/tables/snowflake/nh_link.sql b/macros/tables/snowflake/nh_link.sql
@@ -1,30 +1,33 @@
 {%- macro snowflake__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
-{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
-
-    {%- if execute -%}
-        {{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
-    {%- endif %}
-
-{%- endif -%}
 
 {%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
 
 {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
 {%- set timestamp_format = datavault4dbt.timestamp_format() -%}
+{{ log('source_models: '~source_models, false) }}
 
 {# If no specific link_hk and fk_columns are defined for each source, we apply the values set in the link_hashkey and foreign_hashkeys variable. #}
 {# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed  #}
 {# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
 {%- if source_models is not mapping and not datavault4dbt.is_list(source_models) -%}
     {%- set source_models = {source_models: {}} -%}
 {%- endif -%}
-
 {%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
 {%- set source_models = source_model_values['source_model_list'] -%}
+{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
+{%- for source_model in source_models -%}
+    {%- set source_relation = ref(source_model.name) -%}
+{%- endfor -%}
+
+{%- if execute -%}
+
 {%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
 {%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
 {{ log('source_models: '~source_models, false) }}
 
+{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
+    {%- set foreign_hashkeys = [] -%}
+{%- endif -%}
 {%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
 
 {{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +254,6 @@ records_to_insert AS (
 
 SELECT * FROM records_to_insert
 
+{%- endif -%}
+
 {%- endmacro -%}