From b42e6f94c1cbb296fee17a5e3e281278cfe99a17 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 11 Nov 2021 15:57:31 +1300 Subject: [PATCH 01/13] dbt 0.7.4 release (#441) * Update require-dbt-version to be 1.0 * Fix SQL 42000 on Exasol (#420) " SQL-Error [42000]: syntax error, unexpected '*' " If you specify the * in the unioned with their respectiv names .* you do not receive the SQL Error posted above. This should not inflict any further problems since it is redundant for most DBs. * Minor readme link fixes (#431) * minor readme link fixes * changelog addition Co-authored-by: Joel Labes * 0.7.4 changelog (#432) * Update CHANGELOG.md * Note branch name change * use `limit_zero` macro instead of `limit 0` (#437) * Utils 0.7.4b1 (#433) * Update require-dbt-version to be 1.0 * Fix SQL 42000 on Exasol (#420) " SQL-Error [42000]: syntax error, unexpected '*' " If you specify the * in the unioned with their respectiv names .* you do not receive the SQL Error posted above. This should not inflict any further problems since it is redundant for most DBs. * Minor readme link fixes (#431) * minor readme link fixes * changelog addition Co-authored-by: Joel Labes * 0.7.4 changelog (#432) * Update CHANGELOG.md * Note branch name change Co-authored-by: Timo Kruth Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> * standard convention * Update integration_tests/tests/jinja_helpers/test_slugify.sql Taking the liberty of committing on your behalf so that the CI job starts again * Change limit_zero to be a macro Co-authored-by: Joel Labes Co-authored-by: Timo Kruth Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> * Add col_name alias to else state too (#437) * Remove extra semicolon in `insert_by_period` materialization (#439) * Remove extra semicolon in `insert_by_period` materialization. `create_table_as()` generates a SQL statement that already ends with a semicolon, so the extra semicolon after a `create_table_as()` call in the `insert_by_period` materialization ends up being an empty SQL statement, and at least when using Snowflake this causes the dbt run to fail with a "cannot unpack non-iterable NoneType object" error. * Update changelog for PR 439. * Use the relation object passed into get_column_values, instead of making our own (#440) * Use the relation object passed into get_column_values, instead of making our own * Rename variables in get column value test to be clearer * Update CHANGELOG.md * Update CHANGELOG.md Co-authored-by: Timo Kruth Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Co-authored-by: Anders Co-authored-by: Sean Rose --- CHANGELOG.md | 15 +++++++++++---- .../models/sql/test_get_column_values.sql | 10 +++++----- .../tests/jinja_helpers/test_slugify.sql | 4 ++-- .../insert_by_period_materialization.sql | 2 +- macros/sql/get_column_values.sql | 6 +++--- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6dc7c15..fb5a161e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,17 @@ -# dbt-utils v0.8.0 +# dbt-utils v0.7.4 +🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using dbt-utils 0.7.4 with dbt-core v1.0.0 can expect to see a deprecation warning. This will be resolved in dbt_utils v0.8.0. -# dbt-utils v0.7.4b1 -This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). When dbt-core 1.0.0 hits release candidate status, we will release the final version of 0.7.4 +## Fixes +- `get_column_values()` now works correctly with mixed-quoting styles on Snowflake ([#424](https://github.com/dbt-labs/dbt-utils/issues/424), [#440](https://github.com/dbt-labs/dbt-utils/pull/440)) +- Remove extra semicolon in `insert_by_period` materialization that was causing errors ([#439](https://github.com/dbt-labs/dbt-utils/pull/439)) +- Swap `limit 0` out for `{{ limit_zero() }}` on the `slugify()` tests to allow for compatibility with [tsql-utils](https://github.com/dbt-msft/tsql-utils) ([#437](https://github.com/dbt-labs/dbt-utils/pull/437)) -🚨 Projects using utils 0.7.4 with Core 1.0.0 can expect to see a deprecation warning. This will be resolved in 0.8.0 of dbt_utils alongside the final version of 1.0.0. +## Contributors: +- [sean-rose](https://github.com/sean-rose) +- [@swanderz](https://github.com/swanderz) + +# dbt-utils v0.7.4b1 :rotating_light:🚨 We have renamed the `master` branch to `main`. If you have a local version of `dbt-utils`, you will need to update to the new branch. See the [GitHub docs](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch#updating-a-local-clone-after-a-branch-name-changes) for more details. ## Under the hood diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 8e6f5b45..36214984 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,13 +1,13 @@ -{% set columns = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} {% if target.type == 'snowflake' %} select - {% for column in columns -%} + {% for val in column_values -%} - sum(case when field = '{{ column }}' then 1 else 0 end) as count_{{ column }} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} {%- if not loop.last %},{% endif -%} {%- endfor %} @@ -17,9 +17,9 @@ from {{ ref('data_get_column_values') }} {% else %} select - {% for column in columns -%} + {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ column ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ column }} + {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} {%- if not loop.last %},{% endif -%} {%- endfor %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index 07b184f7..c0839f59 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,7 +1,7 @@ {% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} {# Return 0 rows for the test to pass #} - select 1 limit 0 + select 1 as col_name {{ limit_zero() }} {% else %} {# Return >0 rows for the test to fail #} - select 1 + select 1 as col_name {% endif %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 9b43bdbb..851afa3d 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -102,7 +102,7 @@ {# Create an empty target table -#} {% call statement('main') -%} {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}}; + {{create_table_as(False, target_relation, empty_sql)}} {%- endcall %} {%- endif %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 1f79a455..c2f65aff 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -9,9 +9,9 @@ {{ return('') }} {% endif %} - {%- set target_relation = adapter.get_relation(database=table.database, - schema=table.schema, - identifier=table.identifier) -%} + {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} + {# TODO: Change the method signature in a future 0.x.0 release #} + {%- set target_relation = table -%} {%- call statement('get_column_values', fetch_result=true) %} From 8b38a56b6d92d091155fea5d06d9da8cf8987ff1 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:15:07 +1300 Subject: [PATCH 02/13] Regression: Correctly handle missing relations in get_column_values (#448) * Create integration test for a dropped relation * Update get_column_values.sql * Swap out adapter call for a good old fashioned drop table --- .../sql/data_get_column_values_dropped.csv | 12 +++++++++ integration_tests/dbt_project.yml | 6 +++++ .../test_get_column_values_use_default.sql | 25 +++++++++++++++++++ macros/sql/get_column_values.sql | 11 +++++--- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 integration_tests/data/sql/data_get_column_values_dropped.csv create mode 100644 integration_tests/tests/sql/test_get_column_values_use_default.sql diff --git a/integration_tests/data/sql/data_get_column_values_dropped.csv b/integration_tests/data/sql/data_get_column_values_dropped.csv new file mode 100644 index 00000000..5651149b --- /dev/null +++ b/integration_tests/data/sql/data_get_column_values_dropped.csv @@ -0,0 +1,12 @@ +field +a +b +c +d +e +f +g +g +g +g +g diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 40db9050..9631a8c4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -54,6 +54,12 @@ seeds: sql: data_events_20180103: +schema: events + + data_get_column_values_dropped: + # TODO: Ideally this would use the adapter, but it gives a "Tried to drop relation, but its type is null" error. + #+post-hook: "{% do adapter.drop_relation(this) %}" + +post-hook: "drop table {{ this }}" + schema_tests: data_test_sequential_timestamps: diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql new file mode 100644 index 00000000..6b7569f5 --- /dev/null +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -0,0 +1,25 @@ + +{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} + +with expected as ( + select { dbt_utils.safe_cast('y', dbt_utils.type_string()) }} as expected union all + select { dbt_utils.safe_cast('z', dbt_utils.type_string()) }} as expected +), + +actual as ( + select + {% for val in column_values %} + {{ dbt_utils.safe_cast(val, dbt_utils.type_string()) }} as 'actual' + {% if not loop.last %} + union all + {% endif %} + {% endfor %} +), + +failures as ( + select * from actual + except + select * from expected +) + +select * from failures \ No newline at end of file diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index c2f65aff..61b6bece 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -13,15 +13,18 @@ {# TODO: Change the method signature in a future 0.x.0 release #} {%- set target_relation = table -%} + {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} + {% set relation_exists = (load_relation(target_relation)) is not none %} + {%- call statement('get_column_values', fetch_result=true) %} - {%- if not target_relation and default is none -%} + {%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ table ~ " does not exist and no default value was provided.") }} + {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} - {%- elif not target_relation and default is not none -%} + {%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ table ~ " does not exist. Returning the default value: " ~ default) }} + {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} {{ return(default) }} From 08595370b3f078c82043e78da066e30e61f92050 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:17:17 +1300 Subject: [PATCH 03/13] Add missing curlies --- .../tests/sql/test_get_column_values_use_default.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 6b7569f5..64635e5c 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -2,8 +2,8 @@ {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} with expected as ( - select { dbt_utils.safe_cast('y', dbt_utils.type_string()) }} as expected union all - select { dbt_utils.safe_cast('z', dbt_utils.type_string()) }} as expected + select {{ dbt_utils.safe_cast('y', dbt_utils.type_string()) }} as expected union all + select {{ dbt_utils.safe_cast('z', dbt_utils.type_string()) }} as expected ), actual as ( From 0df84c3c3e3def44fff82ec4ca2b75d27cf7eccb Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:18:56 +1300 Subject: [PATCH 04/13] what person wrote this code :/ (it was me) --- .../tests/sql/test_get_column_values_use_default.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 64635e5c..32e1f659 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -9,7 +9,7 @@ with expected as ( actual as ( select {% for val in column_values %} - {{ dbt_utils.safe_cast(val, dbt_utils.type_string()) }} as 'actual' + {{ dbt_utils.safe_cast(val, dbt_utils.type_string()) }} as actual {% if not loop.last %} union all {% endif %} From 68b187f4a3bb1f66e3a0b416afdbf7a80bf48550 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:22:50 +1300 Subject: [PATCH 05/13] wrap values in quotes --- .../tests/sql/test_get_column_values_use_default.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 32e1f659..ddfd7495 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -9,7 +9,7 @@ with expected as ( actual as ( select {% for val in column_values %} - {{ dbt_utils.safe_cast(val, dbt_utils.type_string()) }} as actual + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual {% if not loop.last %} union all {% endif %} From ef240c7ff2380352de6e0d5ac353c666aaeaa886 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:30:47 +1300 Subject: [PATCH 06/13] GOOD --- .../tests/sql/test_get_column_values_use_default.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index ddfd7495..415ad38d 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -2,14 +2,14 @@ {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} with expected as ( - select {{ dbt_utils.safe_cast('y', dbt_utils.type_string()) }} as expected union all - select {{ dbt_utils.safe_cast('z', dbt_utils.type_string()) }} as expected + select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected union all + select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected ), actual as ( - select + {% for val in column_values %} - {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual + select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual {% if not loop.last %} union all {% endif %} From 614a8fc1c102c1f34a40c393fe054e882c8362af Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:44:40 +1300 Subject: [PATCH 07/13] bigquery compat (they don't like except) --- .../tests/sql/test_get_column_values_use_default.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 415ad38d..7f9fa3a4 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -2,14 +2,14 @@ {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected + select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all + select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value ), actual as ( {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual + select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value {% if not loop.last %} union all {% endif %} @@ -18,8 +18,9 @@ actual as ( failures as ( select * from actual - except - select * from expected + where actual.actual_column_value not in ( + select expected.expected_column_value from expected + ) ) select * from failures \ No newline at end of file From fd8dc17ba9df576fcaef9c16bb0899e528018ba3 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 14:53:41 +1300 Subject: [PATCH 08/13] Backport android url changes from #426 (#452) --- integration_tests/data/web/data_url_host.csv | 5 ++++- macros/web/get_url_host.sql | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/integration_tests/data/web/data_url_host.csv b/integration_tests/data/web/data_url_host.csv index c32fbf8d..2350bb7a 100644 --- a/integration_tests/data/web/data_url_host.csv +++ b/integration_tests/data/web/data_url_host.csv @@ -1,4 +1,7 @@ original_url,parsed_url www.google.co.uk?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer,www.google.co.uk http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,witanddelight.com -https://www.nytimes.com/2018/01/01/blog,www.nytimes.com \ No newline at end of file +https://www.nytimes.com/2018/01/01/blog,www.nytimes.com +android-app://m.facebook.com/,m.facebook.com +docs.nytimes.com/2021/01/01/index.js?utm_source=google,docs.nytimes.com +https://m.facebook.com/,m.facebook.com \ No newline at end of file diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index b1e8e4de..d6662576 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -4,23 +4,24 @@ {% macro default__get_url_host(field) -%} -{%- set parsed = +{%- set parsed = dbt_utils.split_part( dbt_utils.split_part( dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''" + dbt_utils.replace( + dbt_utils.replace(field, "'android-app://'", "''" + ), "'http://'", "''" ), "'https://'", "''" ), "'/'", 1 ), "'?'", 1 ) - + -%} - + {{ dbt_utils.safe_cast( parsed, dbt_utils.type_string() )}} - {%- endmacro %} \ No newline at end of file From db8a047474ffdade6113cbdde7e19c45ff3aae77 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 15:02:45 +1300 Subject: [PATCH 09/13] Update CHANGELOG.md --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb5a161e..139b9c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ -# dbt-utils v0.7.4 +# dbt-utils 0.7.5 🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using dbt-utils 0.7.4 with dbt-core v1.0.0 can expect to see a deprecation warning. This will be resolved in dbt_utils v0.8.0. +## Fixes +- Regression in `get_column_values()` where the default would not be respected if the model didn't exist. ([#444](https://github.com/dbt-labs/dbt-utils/issues/444), [#448](https://github.com/dbt-labs/dbt-utils/pull/448)) + +## Under the hood +- get_url_host() macro now correctly handles URLs beginning with android-app:// (#426) + +## Contributors: +- [foundinblank](https://github.com/foundinblank) + +# dbt-utils v0.7.4 ## Fixes - `get_column_values()` now works correctly with mixed-quoting styles on Snowflake ([#424](https://github.com/dbt-labs/dbt-utils/issues/424), [#440](https://github.com/dbt-labs/dbt-utils/pull/440)) - Remove extra semicolon in `insert_by_period` materialization that was causing errors ([#439](https://github.com/dbt-labs/dbt-utils/pull/439)) From f6b4e4caf71a78c96d3d30de5be654315d575217 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Thu, 2 Dec 2021 15:23:45 +1300 Subject: [PATCH 10/13] Change require-dbt-version, update dbt_project.yml for integration tests proj --- dbt_project.yml | 6 +++--- integration_tests/dbt_project.yml | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index aaaf9d49..0019f15a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,11 +1,11 @@ name: 'dbt_utils' -version: '0.7.0' +version: '0.1.0' -require-dbt-version: [">=0.20.0", "<=1.0.0"] +require-dbt-version: [">=1.0.0-rc", "<2.0.0"] config-version: 2 target-path: "target" -clean-targets: ["target", "dbt_modules"] +clean-targets: ["target", "dbt_modules", "dbt_packages"] macro-paths: ["macros"] log-path: "logs" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9631a8c4..b776a003 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,16 +8,17 @@ profile: 'integration_tests' config-version: 2 -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_modules" + - "dbt_packages" dispatch: - macro_namespace: 'dbt_utils' From a5a0b2f22d5d128223eec36d7cad98e99c707b3a Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Fri, 3 Dec 2021 07:32:39 +1300 Subject: [PATCH 11/13] Upgrade python version in CI, improve drop relation integration test --- .circleci/config.yml | 8 ++++---- integration_tests/dbt_project.yml | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8869c8bb..2432273b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: integration-postgres: docker: - - image: circleci/python:3.6.13-stretch + - image: cimg/python:3.9.9 - image: circleci/postgres:9.6.5-alpine-ram steps: @@ -24,7 +24,7 @@ jobs: integration-redshift: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -35,7 +35,7 @@ jobs: integration-snowflake: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -48,7 +48,7 @@ jobs: environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index b776a003..bb6d147b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -57,9 +57,8 @@ seeds: +schema: events data_get_column_values_dropped: - # TODO: Ideally this would use the adapter, but it gives a "Tried to drop relation, but its type is null" error. - #+post-hook: "{% do adapter.drop_relation(this) %}" - +post-hook: "drop table {{ this }}" + # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet + +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}" schema_tests: From 3cb69fb031fccbd144edf763b7e52c6247b1fa60 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Fri, 3 Dec 2021 07:59:49 +1300 Subject: [PATCH 12/13] Clarify version pinning --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75833b7b..6ad5e3e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # dbt-utils v0.8.0 ## 🚨 Breaking changes -- dbt ONE POINT OH is here! This version of dbt-utils requires `dbt-core ~=1.0.0` (any minor and patch version of v1), which means far less need for compatibility releases in the future. +- dbt ONE POINT OH is here! This version of dbt-utils requires _any_ version (minor and patch) of v1, which means far less need for compatibility releases in the future. - The partition column in the `mutually_exclusive_ranges` test is now always called `partition_by_col`. This enables compatibility with `--store-failures` when multiple columns are concatenated together. If you have models built on top of the failures table, update them to reflect the new column name. ([#423](https://github.com/dbt-labs/dbt-utils/issues/423), [#430](https://github.com/dbt-labs/dbt-utils/pull/430)) ## Contributors: From 77411a9245c3871aa3089869e34b296de5e0183b Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Fri, 3 Dec 2021 10:27:37 +1300 Subject: [PATCH 13/13] Drop support for release candidates of 1.0.0 --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 0019f15a..833501d8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'dbt_utils' version: '0.1.0' -require-dbt-version: [">=1.0.0-rc", "<2.0.0"] +require-dbt-version: [">=1.0.0", "<2.0.0"] config-version: 2