Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change ghost record numeric defaults and add ghost record numeric values variables #251

Merged
merged 4 commits into from
Sep 5, 2024
Merged

Change ghost record numeric defaults and add ghost record numeric values variables #251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c5f9db2
implement global variable to set numeric ghost record values
tkiehn Aug 28, 2024
892da54
change ghost record numeric defaults for unknown, error from 0, -1 t…
tkiehn Sep 5, 2024
f24ebe5
improve postgres ghostrecords
tkiehn Sep 5, 2024
b40a8c8
Merge branch 'main' into add-ghost-record-numeric-values-variables
tkiehn Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ vars:
datavault4dbt.error_value_alt__STRING: 'e'
datavault4dbt.unknown_value__STRING: '(unknown)'
datavault4dbt.unknown_value_alt__STRING: 'u'
datavault4dbt.unknown_value__numeric: '-1'
datavault4dbt.error_value__numeric: '-2'


# Configuring models
Expand Down
73 changes: 47 additions & 26 deletions macros/supporting/ghost_record_per_datatype.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,27 @@

{%- set datatype = datatype | string | upper | trim -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}

{%- if ghost_record_type == 'unknown' -%}
{%- if datatype == 'TIMESTAMP' %} {{ datavault4dbt.string_to_timestamp( timestamp_format , beginning_of_all_times) }} as {{ alias }}
{%- elif datatype == 'DATE'-%} PARSE_DATE('{{date_format}}','{{ beginning_of_all_times_date }}') as {{ alias }}
{%- elif datatype == 'STRING' %} '{{unknown_value__STRING}}' as {{ alias }}
{%- elif datatype == 'INT64' %} CAST('0' as INT64) as {{ alias }}
{%- elif datatype == 'FLOAT64' %} CAST('0' as FLOAT64) as {{ alias }}
{%- elif datatype == 'INT64' %} CAST({{unknown_value__numeric}} as INT64) as {{ alias }}
{%- elif datatype == 'FLOAT64' %} CAST({{unknown_value__numeric}} as FLOAT64) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
{% endif %}
{%- elif ghost_record_type == 'error' -%}
{%- if datatype == 'TIMESTAMP' %} {{ datavault4dbt.string_to_timestamp( timestamp_format , end_of_all_times) }} as {{ alias }}
{%- elif datatype == 'DATE'-%} PARSE_DATE('{{date_format}}', '{{ end_of_all_times_date }}') as {{ alias }}
{%- elif datatype == 'STRING' %} '{{error_value__STRING}}' as {{ alias }}
{%- elif datatype == 'INT64' %} CAST('-1' as INT64) as {{ alias }}
{%- elif datatype == 'FLOAT64' %} CAST('-1' as FLOAT64) as {{ alias }}
{%- elif datatype == 'INT64' %} CAST({{error_value__numeric}} as INT64) as {{ alias }}
{%- elif datatype == 'FLOAT64' %} CAST({{error_value__numeric}} as FLOAT64) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
{% endif %}
Expand All @@ -62,6 +66,9 @@
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
{%- set date_format = datavault4dbt.date_format() -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
{%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%}
Expand Down Expand Up @@ -94,8 +101,8 @@
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as {{ alias }}
{%- endif -%}
{%- elif datatype.upper().startswith('CHAR') -%} CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('0' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('0' as DOUBLE PRECISION) as {{ alias }}
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('{{unknown_value__numeric}}' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('{{unknown_value__numeric}}' as DOUBLE PRECISION) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} FALSE as {{ alias }}
{%- elif datatype.upper().startswith('HASHTYPE') -%} CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
Expand All @@ -122,8 +129,8 @@
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as {{ alias }}
{%- endif -%}
{%- elif datatype.upper().startswith('CHAR') -%} CAST('{{ error_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('-1' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('-1' as DOUBLE PRECISION) as {{ alias }}
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('{{error_value__numeric}}' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('{{error_value__numeric}}' as DOUBLE PRECISION) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} FALSE as {{ alias }}
{%- elif datatype.upper().startswith('HASHTYPE') -%} CAST('{{ error_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
Expand Down Expand Up @@ -151,6 +158,9 @@
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
{%- set date_format = datavault4dbt.date_format() -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
{%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%}
Expand Down Expand Up @@ -180,7 +190,7 @@
{%- else -%}
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as {{ alias }}
{%- endif -%}
{%- elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}0 AS {{ alias }}
{%- elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}{{unknown_value__numeric}} AS {{ alias }}
{%- elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }}
{%- else %}NULL AS {{ alias }}
{% endif %}
Expand All @@ -205,7 +215,7 @@
{%- else -%}
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as {{ alias }}
{%- endif -%}
{% elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}-1 AS {{ alias }}
{% elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}{{error_value__numeric}} AS {{ alias }}
{% elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }}
{% else %}NULL AS {{ alias }}
{% endif %}
Expand All @@ -226,7 +236,9 @@

{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%}
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
{%- set date_format = datavault4dbt.date_format() -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
Expand Down Expand Up @@ -263,7 +275,8 @@
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as "{{ alias }}"
{%- endif -%}
{%- elif datatype == 'TINYINT' -%} CAST('254' as {{ datatype }}) as "{{ alias }}"
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype or datatype == 'BIT'%} CAST('-1' as {{ datatype }}) as "{{ alias }}"
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('{{unknown_value__numeric}}' as {{ datatype }}) as "{{ alias }}"
{%- elif datatype == 'BIT' -%} CAST(0 as {{ datatype }}) as "{{ alias }}"
{%- elif datatype == 'DATE'-%} CONVERT(DATE, '{{ beginning_of_all_times_date }}') as "{{ alias }}"
{%- elif 'BINARY' in datatype -%}
CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as "{{ alias }}"
Expand Down Expand Up @@ -292,7 +305,7 @@
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as "{{ alias }}"
{%- endif -%}
{%- elif datatype == 'TINYINT' -%} CAST('255' as {{ datatype }}) as "{{ alias }}"
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('-2' as {{ datatype }}) as "{{ alias }}"
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('{{error_value__numeric}}' as {{ datatype }}) as "{{ alias }}"
{%- elif datatype == 'BIT' -%} CAST(0 as {{ datatype }}) as "{{ alias }}"
{%- elif datatype == 'DATE'-%} CONVERT(DATE, '{{ end_of_all_times_date }}') as "{{ alias }}"
{%- elif 'BINARY' in datatype -%}
Expand Down Expand Up @@ -320,27 +333,32 @@
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
{%- set date_format = datavault4dbt.date_format() -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
{%- set datatype = datatype | string | upper | trim -%}

{%- if ghost_record_type == 'unknown' -%}
{%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, beginning_of_all_times_date) }} AS {{ alias }}
{%- elif datatype == 'TIME WITH TIME ZONE' %} CAST('00:00:01 UTC' as TIMETZ) as {{ alias }}
{%- elif datatype == 'TIME WITHOUT TIME ZONE' %} CAST('00:00:01' as TIME) as {{ alias }}
{%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
{%- elif datatype == 'TEXT' %} CAST('{{unknown_value__STRING}}' as TEXT) as {{ alias }}
{%- elif datatype == 'VARCHAR' %} CAST('{{unknown_value__STRING}}' as VARCHAR) as {{ alias }}
{%- elif datatype == 'INTEGER' %} CAST('0' as INTEGER) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('0' as DOUBLE PRECISION) as {{ alias }}
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{unknown_value__STRING}}' as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
{% endif %}
{%- elif ghost_record_type == 'error' -%}
{%- elif ghost_record_type == 'error' -%}
{%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, end_of_all_times) }} as {{ alias }}
{%- elif datatype == 'TIME WITH TIME ZONE' %} CAST('23:59:59 UTC' as TIMETZ) as {{ alias }}
{%- elif datatype == 'TIME WITHOUT TIME ZONE' %} CAST('23:59:59' as TIME) as {{ alias }}
{%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
{%- elif datatype == 'TEXT' %} CAST('{{error_value__STRING}}' as TEXT) as {{ alias }}
{%- elif datatype == 'VARCHAR' %} CAST('{{error_value__STRING}}' as VARCHAR) as {{ alias }}
{%- elif datatype == 'INTEGER' %} CAST('-1' as INTEGER) as {{ alias }}
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('-1' as DOUBLE PRECISION) as {{ alias }}
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{error_value__STRING}}' as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
{% endif %}
Expand All @@ -362,6 +380,9 @@
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
{%- set date_format = datavault4dbt.date_format() -%}

{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', -1) -%}
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', -2) -%}

{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}

Expand All @@ -379,8 +400,8 @@
{%- elif datatype == 'TIME' %} CAST('00:00:01' as TIME) as {{ alias }}
{%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{unknown_value__STRING}}' as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(0 as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(0 as {{ datatype }}) as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }}
Expand All @@ -392,8 +413,8 @@
{%- elif datatype == 'TIME' %} CAST('23:59:59' as TIME) as {{ alias }}
{%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{error_value__STRING}}' as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(-1 as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(-1 as {{ datatype }}) as {{ alias }}
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
{%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('FALSE' as BOOLEAN) as {{ alias }}
{%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ error_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
{%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }}
Expand Down