From 07136c95519335b04e9aaa16780834d4f244f0af Mon Sep 17 00:00:00 2001 From: Christine Date: Thu, 21 Jul 2022 12:00:49 -0500 Subject: [PATCH 1/6] Changed the union macro to function with `none` as an argument --- macros/sql/union.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/macros/sql/union.sql b/macros/sql/union.sql index cb960484..89f0b96f 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -85,7 +85,10 @@ ( select + {%- if source_column_name != none %} cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, + {%- endif %} + {% for col_name in ordered_column_names -%} {%- set col = column_superset[col_name] %} From 5c1fb5dfb33166bf1c934a610947bf90dfc9fc23 Mon Sep 17 00:00:00 2001 From: Christine Date: Thu, 21 Jul 2022 12:07:40 -0500 Subject: [PATCH 2/6] Updated the README with the new functionality --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b45c884..83b7e69d 100644 --- a/README.md +++ b/README.md @@ -886,7 +886,8 @@ final query. Note the `include` and `exclude` arguments are mutually exclusive. * `column_override` (optional): A dictionary of explicit column type overrides, e.g. `{"some_field": "varchar(100)"}`.`` * `source_column_name` (optional, `default="_dbt_source_relation"`): The name of -the column that records the source of this row. +the column that records the source of this row. `source_column_name: none` can +be used to omit this column. * `where` (optional): Filter conditions to include in the `where` clause. #### generate_series ([source](macros/sql/generate_series.sql)) From cb215aaddac825aa43b93368cb0cdbf5b97c6f1c Mon Sep 17 00:00:00 2001 From: Christine Date: Thu, 21 Jul 2022 12:22:48 -0500 Subject: [PATCH 3/6] Added to the CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2917bf15..181b25c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ ## Contributors: ---> +# Unreleased + +## New features +- New feature to omit the `source_column_name` column on the `union_relations` macro ([#331](https://github.com/dbt-labs/dbt-utils/issues/331), [#624](https://github.com/dbt-labs/dbt-utils/pull/624)) + +## Contributors: +- [@christineberger](https://github.com/christineberger) (#624) + # dbt-utils v0.8.6 ## New features From 8300eae8b000e778d5e33be5e4150672dd95506e Mon Sep 17 00:00:00 2001 From: Christine Berger Date: Mon, 25 Jul 2022 15:10:12 -0500 Subject: [PATCH 4/6] Update README.md Co-authored-by: Joel Labes --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 83b7e69d..880cc112 100644 --- a/README.md +++ b/README.md @@ -886,8 +886,7 @@ final query. Note the `include` and `exclude` arguments are mutually exclusive. * `column_override` (optional): A dictionary of explicit column type overrides, e.g. `{"some_field": "varchar(100)"}`.`` * `source_column_name` (optional, `default="_dbt_source_relation"`): The name of -the column that records the source of this row. `source_column_name: none` can -be used to omit this column. +the column that records the source of this row. Pass `None` to omit this column from the results. * `where` (optional): Filter conditions to include in the `where` clause. #### generate_series ([source](macros/sql/generate_series.sql)) From 777d3cf9a0ead9a840500eaa0cbada256f960e62 Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 26 Jul 2022 16:20:08 -0500 Subject: [PATCH 5/6] Added integration test for omitting source column on union_relations macro --- integration_tests/models/sql/schema.yml | 5 ++ .../sql/test_union_no_source_column.sql | 6 +++ .../expect_table_columns_to_match_set.sql | 54 +++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 integration_tests/models/sql/test_union_no_source_column.sql create mode 100644 integration_tests/tests/generic/expect_table_columns_to_match_set.sql diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index a74b5d63..5753e770 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -162,6 +162,11 @@ models: - name: favorite_number tests: - dbt_utils.not_constant + + - name: test_union_no_source_column + tests: + - expect_table_columns_to_match_set: + column_list: ["id", "name", "favorite_color", "favorite_number"] - name: test_get_relations_by_pattern tests: diff --git a/integration_tests/models/sql/test_union_no_source_column.sql b/integration_tests/models/sql/test_union_no_source_column.sql new file mode 100644 index 00000000..660bdfee --- /dev/null +++ b/integration_tests/models/sql/test_union_no_source_column.sql @@ -0,0 +1,6 @@ +{{ dbt_utils.union_relations([ + ref('data_union_table_1'), + ref('data_union_table_2') + ], + source_column_name = none +) }} \ No newline at end of file diff --git a/integration_tests/tests/generic/expect_table_columns_to_match_set.sql b/integration_tests/tests/generic/expect_table_columns_to_match_set.sql new file mode 100644 index 00000000..0d487be9 --- /dev/null +++ b/integration_tests/tests/generic/expect_table_columns_to_match_set.sql @@ -0,0 +1,54 @@ +{# + This macro is copied and slightly edited from the dbt_expectations package. + At the time of this addition, dbt_expectations couldn't be added because + integration_tests is installing dbt_utils from local without a hard-coded + path. dbt is not able to resolve duplicate dependencies of dbt_utils + due to this. +#} + +{%- test expect_table_columns_to_match_set(model, column_list, transform="upper") -%} +{%- if execute -%} + {%- set column_list = column_list | map(transform) | list -%} + + {# Replaces dbt_expectations._get_column_list() #} + {%- set relation_column_names = adapter.get_columns_in_relation(model) + | map(attribute="name") + | map(transform) + | list + -%} + + {# Replaces dbt_exoectations._list_intersect() #} + {%- set matching_columns = [] -%} + {%- for itm in column_list -%} + {%- if itm in relation_column_names -%} + {%- do matching_columns.append(itm) -%} + {%- endif -%} + {%- endfor -%} + + with relation_columns as ( + + {% for col_name in relation_column_names %} + select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column + {% if not loop.last %}union all{% endif %} + {% endfor %} + ), + input_columns as ( + + {% for col_name in column_list %} + select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column + {% if not loop.last %}union all{% endif %} + {% endfor %} + ) + select * + from + relation_columns r + full outer join + input_columns i on r.relation_column = i.input_column + where + -- catch any column in input list that is not in the list of table columns + -- or any table column that is not in the input list + r.relation_column is null or + i.input_column is null + +{%- endif -%} +{%- endtest -%} \ No newline at end of file From 96cb89537c79acee9829cd1abceafa6389fa3b8e Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Wed, 27 Jul 2022 10:56:57 +1200 Subject: [PATCH 6/6] Update integration_tests/tests/generic/expect_table_columns_to_match_set.sql --- .../tests/generic/expect_table_columns_to_match_set.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/generic/expect_table_columns_to_match_set.sql b/integration_tests/tests/generic/expect_table_columns_to_match_set.sql index 0d487be9..936cf37c 100644 --- a/integration_tests/tests/generic/expect_table_columns_to_match_set.sql +++ b/integration_tests/tests/generic/expect_table_columns_to_match_set.sql @@ -17,7 +17,7 @@ | list -%} - {# Replaces dbt_exoectations._list_intersect() #} + {# Replaces dbt_expectations._list_intersect() #} {%- set matching_columns = [] -%} {%- for itm in column_list -%} {%- if itm in relation_column_names -%}