From 8321cb2f69f4e1840f8ea8c67d0adcd14b058d37 Mon Sep 17 00:00:00 2001 From: Tim Kirschke <81677440+tkirschke@users.noreply.github.com> Date: Fri, 7 Jun 2024 08:44:41 +0200 Subject: [PATCH 1/3] Removed double comma --- macros/staging/hash_columns.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/macros/staging/hash_columns.sql b/macros/staging/hash_columns.sql index 4c74ce40..f569493d 100644 --- a/macros/staging/hash_columns.sql +++ b/macros/staging/hash_columns.sql @@ -21,8 +21,6 @@ is_hashdiff=columns[col]['is_hashdiff'], multi_active_key=multi_active_key, main_hashkey_column=main_hashkey_column) -}} - - {{- ", \n" if not loop.last -}} {#- Apply standard hashing for hash key attributes. -#} {%- elif columns[col] is not mapping -%} From 6c25fa214096ee80d88bcdf67311e93841c1b255 Mon Sep 17 00:00:00 2001 From: Tim Kirschke <81677440+tkirschke@users.noreply.github.com> Date: Fri, 7 Jun 2024 08:58:33 +0200 Subject: [PATCH 2/3] Escaped Column Names in Ghost Records --- .../supporting/ghost_record_per_datatype.sql | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/macros/supporting/ghost_record_per_datatype.sql b/macros/supporting/ghost_record_per_datatype.sql index 72f3be18..27a22a82 100644 --- a/macros/supporting/ghost_record_per_datatype.sql +++ b/macros/supporting/ghost_record_per_datatype.sql @@ -156,12 +156,14 @@ {%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%} {%- set error_value_alt__STRING = var('datavault4dbt.error_value_alt__STRING', 'e') -%} {%- set datatype = datatype | string | upper | trim -%} + +{%- set alias = datavault4dbt.escape_column_names(alias) -%} {%- if ghost_record_type == 'unknown' -%} {%- if datatype in ['TIMESTAMP_NTZ','TIMESTAMP'] %}{{ datavault4dbt.string_to_timestamp(timestamp_format, beginning_of_all_times) }} AS {{ alias }} - {%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as "{{ alias }}" + {%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }} {%- elif datatype in ['STRING', 'VARCHAR'] %}'{{ unknown_value__STRING }}' AS {{ alias }} - {%- elif datatype == 'CHAR' %}CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }} ) as "{{ alias }}" + {%- elif datatype == 'CHAR' %}CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }} ) as {{ alias }} {%- elif datatype.upper().startswith('VARCHAR(') or datatype.upper().startswith('CHAR(') -%} {%- if col_size is not none -%} {%- set unknown_dtype_length = col_size | int -%} @@ -174,9 +176,9 @@ {%- set unknown_dtype_length = inside_parenthesis | int -%} {%- endif -%} {%- if unknown_dtype_length < unknown_value__STRING|length -%} - CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }} ) as "{{ alias }}" + CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }} ) as {{ alias }} {%- else -%} - CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as "{{ alias }}" + CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as {{ alias }} {%- endif -%} {%- elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}0 AS {{ alias }} {%- elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }} @@ -184,9 +186,9 @@ {% endif %} {%- elif ghost_record_type == 'error' -%} {%- if datatype in ['TIMESTAMP_NTZ','TIMESTAMP'] %}{{ datavault4dbt.string_to_timestamp(timestamp_format, end_of_all_times) }} AS {{ alias }} - {%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as "{{ alias }}" + {%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }} {%- elif datatype in ['STRING','VARCHAR'] %}'{{ error_value__STRING }}' AS {{ alias }} - {%- elif datatype == 'CHAR' %}CAST('{{ error_value_alt__STRING }}' as {{ datatype }} ) as "{{ alias }}" + {%- elif datatype == 'CHAR' %}CAST('{{ error_value_alt__STRING }}' as {{ datatype }} ) as {{ alias }} {%- elif datatype.upper().startswith('VARCHAR(') or datatype.upper().startswith('CHAR(') -%} {%- if col_size is not none -%} {%- set error_dtype_length = col_size | int -%} @@ -199,9 +201,9 @@ {%- set error_dtype_length = inside_parenthesis | int -%} {%- endif -%} {%- if error_dtype_length < error_value__STRING|length -%} - CAST('{{ error_value_alt__STRING }}' as {{ datatype }} ) as "{{ alias }}" + CAST('{{ error_value_alt__STRING }}' as {{ datatype }} ) as {{ alias }} {%- else -%} - CAST('{{ error_value__STRING }}' as {{ datatype }} ) as "{{ alias }}" + CAST('{{ error_value__STRING }}' as {{ datatype }} ) as {{ alias }} {%- endif -%} {% elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}-1 AS {{ alias }} {% elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }} From 3112c410e98cd83e72fa669876f2d39391f4f911 Mon Sep 17 00:00:00 2001 From: Tim Kirschke <81677440+tkirschke@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:04:12 +0200 Subject: [PATCH 3/3] Fixed Processing of Columns for Case Insensitivity --- macros/internal/helpers/stage_processing_macros.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/macros/internal/helpers/stage_processing_macros.sql b/macros/internal/helpers/stage_processing_macros.sql index ada17e92..daa2402b 100644 --- a/macros/internal/helpers/stage_processing_macros.sql +++ b/macros/internal/helpers/stage_processing_macros.sql @@ -1,5 +1,6 @@ {%- macro process_columns_to_select(columns_list=none, exclude_columns_list=none) -%} {% set exclude_columns_list = exclude_columns_list | map('upper') | list %} + {% set columns_list = columns_list | map('upper') | list %} {% set columns_to_select = [] %} {% if not datavault4dbt.is_list(columns_list) or not datavault4dbt.is_list(exclude_columns_list) %}