From a7290442b401cdfb930d1f527cb9782022d867d9 Mon Sep 17 00:00:00 2001 From: Claire Carroll Date: Sun, 6 Jun 2021 15:08:30 -0400 Subject: [PATCH] 0.7.0 into main (#372) * Tidy up changelog * Add 0.7.0 entry to changelog * Add order_by argument to get_column_values (#349) * Add slugify macro to utils, use in pivot macro (#314) * 0.20.0 compatibility (#371) * Explicitly redefine Redshift -> default * Upgrade generic tests * Rm namespaces macro. New dispatch syntax * Run tests with 0.20.0rc1 * Update changelog, readme Co-authored-by: Jeremy Cohen * Simplify concat (#373) * Postgres also have an alternative concat binary operation (#296) * Update default implementation of concat macro Co-authored-by: Christophe Duong Co-authored-by: Jeremy Cohen Co-authored-by: Christophe Duong --- .circleci/config.yml | 2 +- CHANGELOG.md | 67 ++++++++++-- README.md | 101 ++++++++++++++---- dbt_project.yml | 3 +- integration_tests/dbt_project.yml | 5 +- integration_tests/macros/limit_zero.sql | 6 +- integration_tests/macros/tests.sql | 14 ++- .../models/schema_tests/schema.yml | 1 - integration_tests/models/sql/schema.yml | 2 +- .../models/sql/test_get_column_values.sql | 2 +- .../assert_pretty_output_msg_is_string.sql | 0 .../assert_pretty_time_is_string.sql | 0 .../tests/jinja_helpers/test_slugify.sql | 7 ++ .../cross_db_utils/_get_utils_namespaces.sql | 4 - macros/cross_db_utils/cast_bool_to_text.sql | 2 +- macros/cross_db_utils/concat.sql | 18 +--- macros/cross_db_utils/current_timestamp.sql | 9 +- macros/cross_db_utils/datatypes.sql | 12 +-- macros/cross_db_utils/date_trunc.sql | 2 +- macros/cross_db_utils/dateadd.sql | 9 +- macros/cross_db_utils/datediff.sql | 10 +- macros/cross_db_utils/except.sql | 2 +- macros/cross_db_utils/hash.sql | 2 +- macros/cross_db_utils/identifier.sql | 2 +- macros/cross_db_utils/intersect.sql | 2 +- macros/cross_db_utils/last_day.sql | 9 +- macros/cross_db_utils/length.sql | 2 +- macros/cross_db_utils/literal.sql | 2 +- macros/cross_db_utils/position.sql | 2 +- macros/cross_db_utils/replace.sql | 2 +- macros/cross_db_utils/right.sql | 2 +- macros/cross_db_utils/safe_cast.sql | 2 +- macros/cross_db_utils/split_part.sql | 2 +- macros/cross_db_utils/width_bucket.sql | 2 +- macros/{logger => jinja_helpers}/log_info.sql | 2 +- .../pretty_log_format.sql | 2 +- .../{logger => jinja_helpers}/pretty_time.sql | 2 +- macros/jinja_helpers/slugify.sql | 12 +++ .../insert_by_period_materialization.sql | 4 +- macros/schema_tests/accepted_range.sql | 17 +-- macros/schema_tests/at_least_one.sql | 14 ++- macros/schema_tests/cardinality_equality.sql | 17 ++- macros/schema_tests/equal_rowcount.sql | 26 ++--- macros/schema_tests/equality.sql | 36 +++---- macros/schema_tests/expression_is_true.sql | 35 +++--- macros/schema_tests/fewer_rows_than.sql | 28 ++--- .../mutually_exclusive_ranges.sql | 8 +- macros/schema_tests/not_accepted_values.sql | 12 +-- macros/schema_tests/not_constant.sql | 27 ++--- macros/schema_tests/recency.sql | 29 +++-- macros/schema_tests/relationships_where.sql | 13 +-- macros/schema_tests/sequential_values.sql | 11 +- macros/schema_tests/test_not_null_where.sql | 25 ++--- macros/schema_tests/test_unique_where.sql | 34 ++---- .../unique_combination_of_columns.sql | 17 ++- macros/sql/date_spine.sql | 4 +- macros/sql/generate_series.sql | 4 +- macros/sql/get_column_values.sql | 24 ++--- macros/sql/get_query_results_as_dict.sql | 2 +- macros/sql/get_relations_by_pattern.sql | 2 +- macros/sql/get_relations_by_prefix.sql | 2 +- macros/sql/get_tables_by_pattern_sql.sql | 2 +- macros/sql/get_tables_by_prefix_sql.sql | 2 +- macros/sql/groupby.sql | 2 +- macros/sql/haversine_distance.sql | 2 +- macros/sql/nullcheck.sql | 2 +- macros/sql/nullcheck_table.sql | 2 +- macros/sql/pivot.sql | 4 +- macros/sql/safe_add.sql | 2 +- macros/sql/star.sql | 2 +- macros/sql/surrogate_key.sql | 2 +- macros/sql/union.sql | 2 +- macros/sql/unpivot.sql | 2 +- macros/web/get_url_host.sql | 2 +- macros/web/get_url_parameter.sql | 2 +- macros/web/get_url_path.sql | 2 +- run_test.sh | 2 +- 77 files changed, 419 insertions(+), 333 deletions(-) rename integration_tests/tests/{logger => jinja_helpers}/assert_pretty_output_msg_is_string.sql (100%) rename integration_tests/tests/{logger => jinja_helpers}/assert_pretty_time_is_string.sql (100%) create mode 100644 integration_tests/tests/jinja_helpers/test_slugify.sql delete mode 100644 macros/cross_db_utils/_get_utils_namespaces.sql rename macros/{logger => jinja_helpers}/log_info.sql (61%) rename macros/{logger => jinja_helpers}/pretty_log_format.sql (61%) rename macros/{logger => jinja_helpers}/pretty_time.sql (64%) create mode 100644 macros/jinja_helpers/slugify.sql diff --git a/.circleci/config.yml b/.circleci/config.yml index 230824acc88..8869c8bb3e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: integration-postgres: docker: - - image: circleci/python:3.6.3-stretch + - image: circleci/python:3.6.13-stretch - image: circleci/postgres:9.6.5-alpine-ram steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ddf1b68fc..9d8720ffacb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,27 +1,76 @@ +# dbt-utils v0.7.0 (unreleased) +## Breaking changes + +### 🚨 New dbt version + +dbt v0.20.0 or greater is required for this release. If you are not ready to upgrade, consider using a previous release of this package. In accordance with the version upgrade, this package release includes breaking changes to: +- Generic (schema) tests +- `dispatch` functionality + +### 🚨 get_column_values +The order of (optional) arguments has changed in the `get_column_values` macro. + +Before: +```jinja +{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} +... +{% endmacro %} +``` + +After: +```jinja +{% macro get_column_values(table, column, max_records=none, default=none) -%} +... +{% endmacro %} +``` +If you were relying on the position to match up your optional arguments, this may be a breaking change — in general, we recommend that you explicitly declare any optional arguments (if not all of your arguments!) +``` +-- before: This works on previous version of dbt-utils, but on 0.7.0, the `50` would be passed through as the `order_by` argument +{% set payment_methods = dbt_utils.get_column_values( + ref('stg_payments'), + 'payment_method', + 50 +) %} + +-- after +{% set payment_methods = dbt_utils.get_column_values( + ref('stg_payments'), + 'payment_method', + max_records=50 +) %} +``` + +## Features +* Add new argument, `order_by`, to `get_column_values` (code originally in [#289](https://github.com/fishtown-analytics/dbt-utils/pull/289/) from [@clausherther](https://github.com/clausherther), merged via [#349](https://github.com/fishtown-analytics/dbt-utils/pull/349/)) +* Add `slugify` macro, and use it in the pivot macro. :rotating_light: This macro uses the `re` module, which is only available in dbt v0.19.0+. As a result, this feature introduces a breaking change. ([#314](https://github.com/fishtown-analytics/dbt-utils/pull/314)) + +## Under the hood +* Update the default implementation of concat macro to use `||` operator ([#373](https://github.com/fishtown-analytics/dbt-utils/pull/314) [@ChristopheDuong](https://github.com/ChristopheDuong)). Note this may be a breaking change for spark users. + # dbt-utils v0.6.6 ## Fixes - make `sequential_values` schema test use `dbt_utils.type_timestamp()` to allow for compatibility with db's without timestamp data type. [#376](https://github.com/fishtown-analytics/dbt-utils/pull/376) from [@swanderz](https://github.com/swanderz) + # dbt-utils v0.6.5 ## Features * Add new `accepted_range` test ([#276](https://github.com/fishtown-analytics/dbt-utils/pull/276) [@joellabes](https://github.com/joellabes)) -* Make `expression_is_true` work as a column test (code originally in [#226](https://github.com/fishtown-analytics/dbt-utils/pull/226/) from [@elliottohara](https://github.com/elliottohara), merged via [#313]) +* Make `expression_is_true` work as a column test (code originally in [#226](https://github.com/fishtown-analytics/dbt-utils/pull/226/) from [@elliottohara](https://github.com/elliottohara), merged via [#313](https://github.com/fishtown-analytics/dbt-utils/pull/313/)) * Add new schema test, `not_accepted_values` ([#284](https://github.com/fishtown-analytics/dbt-utils/pull/284) [@JavierMonton](https://github.com/JavierMonton)) -* Support a new argument, `zero_length_range_allowed` in the `mutually_exclusive_ranges` test ([#307](https://github.com/fishtown-analytics/dbt-utils/pull/307) [@zemekeng](https://github.com/zemekeneng)) +* Support a new argument, `zero_length_range_allowed` in the `mutually_exclusive_ranges` test ([#307](https://github.com/fishtown-analytics/dbt-utils/pull/307) [@zemekeneng](https://github.com/zemekeneng)) * Add new schema test, `sequential_values` ([#318](https://github.com/fishtown-analytics/dbt-utils/pull/318), inspired by [@hundredwatt](https://github.com/hundredwatt)) -* Support `quarter` in the `postgres__last_day` macro ([#333](https://github.com/fishtown-analytics/dbt-utils/pull/333/files), [@seunghanhong](https://github.com/seunghanhong)) -* Add new argument, `unit`, to `haversine_distance` [#340](https://github.com/fishtown-analytics/dbt-utils/pull/340) [@bastienboutonnet](https://github.com/bastienboutonnet) -* Add new schema test, `fewer_rows_than` (code originally in [#221](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@dmarts](https://github.com/dmarts), merged via [#343]) - +* Support `quarter` in the `postgres__last_day` macro ([#333](https://github.com/fishtown-analytics/dbt-utils/pull/333/files) [@seunghanhong](https://github.com/seunghanhong)) +* Add new argument, `unit`, to `haversine_distance` ([#340](https://github.com/fishtown-analytics/dbt-utils/pull/340) [@bastienboutonnet](https://github.com/bastienboutonnet)) +* Add new schema test, `fewer_rows_than` (code originally in [#221](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@dmarts](https://github.com/dmarts), merged via [#343](https://github.com/fishtown-analytics/dbt-utils/pull/343/)) ## Fixes * Handle booleans gracefully in the unpivot macro ([#305](https://github.com/fishtown-analytics/dbt-utils/pull/305) [@avishalom](https://github.com/avishalom)) -* Fix a bug in `get_relation_by_prefix` that happens with Snowflake external tables. Now the macro will retrieve tables that match the prefix which are external tables ([#350](https://github.com/fishtown-analytics/dbt-utils/issues/350)) -* Fix `cardinality_equality` test when the two tables' column names differed ([#334](https://github.com/fishtown-analytics/dbt-utils/pull/334)) [@joellabes](https://github.com/joellabes) +* Fix a bug in `get_relation_by_prefix` that happens with Snowflake external tables. Now the macro will retrieve tables that match the prefix which are external tables ([#351](https://github.com/fishtown-analytics/dbt-utils/pull/351)) +* Fix `cardinality_equality` test when the two tables' column names differed ([#334](https://github.com/fishtown-analytics/dbt-utils/pull/334) [@joellabes](https://github.com/joellabes)) ## Under the hood -* Fix Markdown formatting for hub rendering ([#336](https://github.com/fishtown-analytics/dbt-utils/issues/350), [@coapacetic](https://github.com/coapacetic)) +* Fix Markdown formatting for hub rendering ([#336](https://github.com/fishtown-analytics/dbt-utils/issues/350) [@coapacetic](https://github.com/coapacetic)) * Reorder readme and improve docs # dbt-utils v0.6.4 diff --git a/README.md b/README.md index d3d9a9874cb..8a805326cdf 100644 --- a/README.md +++ b/README.md @@ -510,8 +510,14 @@ These macros run a query and return the results of the query as objects. They ar #### get_column_values ([source](macros/sql/get_column_values.sql)) -This macro returns the unique values for a column in a given [relation](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation). -It takes an options `default` argument for compiling when the relation does not already exist. +This macro returns the unique values for a column in a given [relation](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) as an array. + +Arguments: +- `table` (required): a [Relation](https://docs.getdbt.com/reference/dbt-classes#relation) (a `ref` or `source`) that contains the list of columns you wish to select from +- `column` (required): The name of the column you wish to find the column values of +- `order_by` (optional, default=`'count(*) desc'`): How the results should be ordered. The default is to order by `count(*) desc`, i.e. decreasing frequency. Setting this as `'my_column'` will sort alphabetically, while `'min(created_at)'` will sort by when thevalue was first observed. +- `max_records` (optional, default=`none`): The maximum number of column values you want to return +- `default` (optional, default=`[]`): The results this macro should return if the relation has not yet been created (and therefore has no column values). **Usage:** @@ -519,15 +525,35 @@ It takes an options `default` argument for compiling when the relation does not -- Returns a list of the payment_methods in the stg_payments model_ {% set payment_methods = dbt_utils.get_column_values(table=ref('stg_payments'), column='payment_method') %} -{% for state in states %} +{% for payment_method in payment_methods %} ... {% endfor %} ... ``` -#### get_relations_by_pattern ([source](macros/sql/get_relations_by_pattern.sql)) +```sql +-- Returns the list sorted alphabetically +{% set payment_methods = dbt_utils.get_column_values( + table=ref('stg_payments'), + column='payment_method', + order_by='payment_method' +) %} +``` + +```sql +-- Returns the list sorted my most recently observed +{% set payment_methods = dbt_utils.get_column_values( + table=ref('stg_payments'), + column='payment_method', + order_by='max(created_at) desc', + max_records=50, + default=['bank_transfer', 'coupon', 'credit_card'] +%} +... +``` +#### get_relations_by_pattern ([source](macros/sql/get_relations_by_pattern.sql)) Returns a list of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) that match a given schema- or table-name pattern. @@ -948,8 +974,8 @@ When an expression falls outside the range, the function returns: --- -### Logger -#### pretty_time ([source](macros/logger/pretty_time.sql)) +### Jinja Helpers +#### pretty_time ([source](macros/jinja_helpers/pretty_time.sql)) This macro returns a string of the current timestamp, optionally taking a datestring format. ```sql {#- This will return a string like '14:50:34' -#} @@ -959,7 +985,7 @@ This macro returns a string of the current timestamp, optionally taking a datest {{ dbt_utils.pretty_time(format='%Y-%m-%d %H:%M:%S') }} ``` -#### pretty_log_format ([source](macros/logger/pretty_log_format.sql)) +#### pretty_log_format ([source](macros/jinja_helpers/pretty_log_format.sql)) This macro formats the input in a way that will print nicely to the command line when you `log` it. ```sql {#- This will return a string like: @@ -968,7 +994,7 @@ This macro formats the input in a way that will print nicely to the command line {{ dbt_utils.pretty_log_format("my pretty message") }} ``` -#### log_info ([source](macros/logger/log_info.sql)) +#### log_info ([source](macros/jinja_helpers/log_info.sql)) This macro logs a formatted message (with a timestamp) to the command line. ```sql {{ dbt_utils.log_info("my pretty message") }} @@ -979,6 +1005,40 @@ This macro logs a formatted message (with a timestamp) to the command line. 11:07:31 + my pretty message ``` +#### slugify ([source](macros/jinja_helpers/slugify.sql)) +This macro is useful for transforming Jinja strings into "slugs", and can be useful when using a Jinja object as a column name, especially when that Jinja object is not hardcoded. + +For this example, let's pretend that we have payment methods in our payments table like `['venmo App', 'ca$h-money']`, which we can't use as a column name due to the spaces and special characters. This macro does its best to strip those out in a sensible way: `['venmo_app', +'cah_money']`. + +```sql +{%- set payment_methods = dbt_utils.get_column_values( + table=ref('raw_payments'), + column='payment_method' +) -%} + +select +order_id, +{%- for payment_method in payment_methods %} +sum(case when payment_method = '{{ payment_method }}' then amount end) + as {{ slugify(payment_method) }}_amount, + +{% endfor %} +... +``` + +```sql +select +order_id, + +sum(case when payment_method = 'Venmo App' then amount end) + as venmo_app_amount, + +sum(case when payment_method = 'ca$h money' then amount end) + as cah_money_amount, +... +``` + ### Materializations #### insert_by_period ([source](macros/materializations/insert_by_period_materialization.sql)) `insert_by_period` allows dbt to insert records into a table one period (i.e. day, week) at a time. @@ -1046,29 +1106,26 @@ We welcome contributions to this repo! To contribute a new feature or a fix, ple **Note:** This is primarily relevant to: - Users and maintainers of community-supported [adapter plugins](https://docs.getdbt.com/docs/available-adapters) - Users who wish to override a low-lying `dbt_utils` macro with a custom implementation, and have that implementation used by other `dbt_utils` macros + If you use Postgres, Redshift, Snowflake, or Bigquery, this likely does not apply to you. -dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different implementations of the same macro -across different databases. +dbt v0.18.0 introduced [`adapter.dispatch()`](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter#dispatch), a reliable way to define different implementations of the same macro across different databases. -All dispatched macros in `dbt_utils` have an override setting: a `var` named -`dbt_utils_dispatch_list` that accepts a list of package names. If you set this -variable in your project, when dbt searches for implementations of a dispatched -`dbt_utils` macro, it will search through your listed packages _before_ using -the implementations defined in `dbt_utils`. +dbt v0.20.0 introduced a new project-level `dispatch` config that enables an "override" setting for all dispatched macros. If you set this config in your project, when dbt searches for implementations of a macro in the `dbt_utils` namespace, it will search through your list of packages instead of just looking in the `dbt_utils` package. -Set a variable in your `dbt_project.yml`: +Set the config in `dbt_project.yml`: ```yml -vars: - dbt_utils_dispatch_list: - - first_package_to_search # likely the name of your root project (only the root folder) - - second_package_to_search # likely an "add-on" package, such as spark_utils - # dbt_utils is always the last place searched +dispatch: + - macro_namespace: dbt_utils + search_order: + - first_package_to_search # likely the name of your root project + - second_package_to_search # could be a "shim" package, such as spark_utils + - dbt_utils # always include dbt_utils as the last place to search ``` If overriding a dispatched macro with a custom implementation in your own project's `macros/` directory, you must name your custom macro with a prefix: either `default__` (note the two underscores), or the name of your adapter followed by two underscores. For example, if you're running on Postgres and wish to override the behavior of `dbt_utils.datediff` (such that `dbt_utils.date_spine` will use your version instead), you can do this by defining a macro called either `default__datediff` or `postgres__datediff`. -When running on Spark, if dbt needs to dispatch `dbt_utils.datediff`, it will search for the following in order: +Let's say we have the config defined above, and we're running on Spark. When dbt goes to dispatch `dbt_utils.datediff`, it will search for macros the following in order: ``` first_package_to_search.spark__datediff first_package_to_search.default__datediff diff --git a/dbt_project.yml b/dbt_project.yml index 7da94610d3e..e5d5f93ac88 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,8 @@ name: 'dbt_utils' version: '0.1.0' -require-dbt-version: [">=0.18.0", "<0.20.0"] +require-dbt-version: [">=0.20.0", "<0.21.0"] + config-version: 2 target-path: "target" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4b5b10eb7e8..40db90501c4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -19,8 +19,9 @@ clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_modules" -vars: - dbt_utils_dispatch_list: ['dbt_utils_integration_tests'] +dispatch: + - macro_namespace: 'dbt_utils' + search_order: ['dbt_utils_integration_tests', 'dbt_utils'] seeds: diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index c37294f0893..697849d1799 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,5 +1,9 @@ +{% macro my_custom_macro() %} + whatever +{% endmacro %} + {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} {% endmacro %} {% macro default__limit_zero() %} diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql index caeb5d6e2eb..046fab3a048 100644 --- a/integration_tests/macros/tests.sql +++ b/integration_tests/macros/tests.sql @@ -1,14 +1,12 @@ -{% macro test_assert_equal(model, actual, expected) %} -select count(*) from {{ model }} where {{ actual }} != {{ expected }} +{% test assert_equal(model, actual, expected) %} +select * from {{ model }} where {{ actual }} != {{ expected }} -{% endmacro %} +{% endtest %} -{% macro test_not_empty_string(model, arg) %} +{% test not_empty_string(model, column_name) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} +select * from {{ model }} where {{ column_name }} = '' -select count(*) from {{ model }} where {{ column_name }} = '' - -{% endmacro %} +{% endtest %} diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 885586a9a6e..25ce3ac888c 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -86,7 +86,6 @@ models: - name: id tests: - dbt_utils.relationships_where: - from: id to: ref('data_test_relationships_where_table_1') field: id from_condition: id <> 4 diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index 07ce68ba394..0e8b24975c4 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -4,7 +4,7 @@ models: - name: test_generate_series tests: - dbt_utils.equality: - arg: ref('data_generate_series') + compare_model: ref('data_generate_series') - name: test_get_column_values columns: diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index bfe5c486383..8e6f5b4509f 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,5 +1,5 @@ -{% set columns = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default = []) %} +{% set columns = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} {% if target.type == 'snowflake' %} diff --git a/integration_tests/tests/logger/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql similarity index 100% rename from integration_tests/tests/logger/assert_pretty_output_msg_is_string.sql rename to integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql diff --git a/integration_tests/tests/logger/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql similarity index 100% rename from integration_tests/tests/logger/assert_pretty_time_is_string.sql rename to integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql new file mode 100644 index 00000000000..07b184f7cb5 --- /dev/null +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -0,0 +1,7 @@ +{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} + {# Return 0 rows for the test to pass #} + select 1 limit 0 +{% else %} + {# Return >0 rows for the test to fail #} + select 1 +{% endif %} diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql deleted file mode 100644 index e841381c415..00000000000 --- a/macros/cross_db_utils/_get_utils_namespaces.sql +++ /dev/null @@ -1,4 +0,0 @@ -{% macro _get_utils_namespaces() %} - {% set override_namespaces = var('dbt_utils_dispatch_list', []) %} - {% do return(override_namespaces + ['dbt_utils']) %} -{% endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 522216ebd9b..5efa9d60b4f 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,5 +1,5 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', packages = dbt_utils._get_utils_namespaces()) (field) }} + {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 32443cbdcc3..7e97e8f8d70 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,23 +1,9 @@ {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', packages = dbt_utils._get_utils_namespaces())(fields)) }} + {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} {%- endmacro %} {% macro default__concat(fields) -%} - concat({{ fields|join(', ') }}) -{%- endmacro %} - -{% macro alternative_concat(fields) %} {{ fields|join(' || ') }} -{% endmacro %} - - -{% macro redshift__concat(fields) %} - {{ dbt_utils.alternative_concat(fields) }} -{% endmacro %} - - -{% macro snowflake__concat(fields) %} - {{ dbt_utils.alternative_concat(fields) }} -{% endmacro %} +{%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 477181d1c05..66ad8dc6ba6 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,5 +1,5 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} {%- endmacro %} {% macro default__current_timestamp() %} @@ -17,7 +17,7 @@ {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} @@ -31,3 +31,8 @@ {% macro postgres__current_timestamp_in_utc() %} (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} {% endmacro %} + +{# redshift should use default instead of postgres #} +{% macro redshift__current_timestamp_in_utc() %} + {{ return(dbt_utils.default__current_timestamp_in_utc()) }} +{% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index 6cb71598085..3dc532b8563 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,7 +1,7 @@ {# string ------------------------------------------------- #} {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_string() %} @@ -25,7 +25,7 @@ {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_timestamp() %} @@ -40,7 +40,7 @@ {# float ------------------------------------------------- #} {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_float() %} @@ -54,7 +54,7 @@ {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_numeric() %} @@ -69,7 +69,7 @@ {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_bigint() %} @@ -83,7 +83,7 @@ {# int ------------------------------------------------- #} {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_int() %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index 83c52dea8cd..cba3346b191 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,5 +1,5 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', packages = dbt_utils._get_utils_namespaces()) (datepart, date)) }} + {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 71a882800d8..09c0f1159b7 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,5 +1,5 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', packages = dbt_utils._get_utils_namespaces())(datepart, interval, from_date_or_timestamp)) }} + {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} {% endmacro %} @@ -28,3 +28,10 @@ {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} + +{# redshift should use default instead of postgres #} +{% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} + + {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} + +{% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index ab5559d7cb6..42dd738eb9b 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,5 +1,5 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', packages = dbt_utils._get_utils_namespaces())(first_date, second_date, datepart)) }} + {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} {% endmacro %} @@ -56,3 +56,11 @@ {% endif %} {% endmacro %} + + +{# redshift should use default instead of postgres #} +{% macro redshift__datediff(first_date, second_date, datepart) %} + + {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} + +{% endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 4ccf7fdd00f..9bb75b86de2 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ return(adapter.dispatch('except', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('except', 'dbt_utils')()) }} {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index fdb293ed0a8..d086c2e63db 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,5 +1,5 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', packages = dbt_utils._get_utils_namespaces()) (field)) }} + {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index ac790b51ce9..619cb7cb787 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -4,7 +4,7 @@ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ '.format(model.package_name, model.name) -%} {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', packages = dbt_utils._get_utils_namespaces()) (value)) }} + {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} {% endmacro %} {% macro default__identifier(value) -%} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index 50bd320b03f..b53c72f38e9 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', packages = dbt_utils._get_utils_namespaces())()) }} + {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 160ec2e12b8..18ca161c4f8 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -4,7 +4,7 @@ testing is required to validate that it will work on other dateparts. */ {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', packages = dbt_utils._get_utils_namespaces()) (date, datepart)) }} + {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} {% endmacro %} @@ -36,3 +36,10 @@ testing is required to validate that it will work on other dateparts. {%- endif -%} {%- endmacro %} + +{# redshift should use default instead of postgres #} +{% macro redshift__last_day(date, datepart) %} + + {{ return(dbt_utils.default__last_day(date, datepart)) }} + +{% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 943b3b610bd..8c2e265c200 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,5 +1,5 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', packages = dbt_utils._get_utils_namespaces()) (expression)) }} + {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} {% endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index 22a3f17a7af..c22914676b3 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,6 +1,6 @@ {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', packages = dbt_utils._get_utils_namespaces()) (value)) }} + {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} {%- endmacro -%} {% macro default__string_literal(value) -%} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index e3f06c248f2..a67b7aa2f6c 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,5 +1,5 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', packages = dbt_utils._get_utils_namespaces()) (substring_text, string_text)) }} + {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} {% endmacro %} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 4f2a194ffaa..ef27d8fcf5e 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,5 +1,5 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', packages = dbt_utils._get_utils_namespaces()) (field, old_chars, new_chars)) }} + {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} {% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 9b34cf10867..8ae1640f6fb 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,5 +1,5 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', packages = dbt_utils._get_utils_namespaces()) (string_text, length_expression)) }} + {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 2eb35c90ae3..8569eecd8dc 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,5 +1,5 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', packages = dbt_utils._get_utils_namespaces()) (field, type)) }} + {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index f44397cd226..036f7d5a6e0 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,5 +1,5 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', packages = dbt_utils._get_utils_namespaces()) (string_text, delimiter_text, part_number)) }} + {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 41978d67ed4..fc8ae81cc5e 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,5 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', packages = dbt_utils._get_utils_namespaces()) (expr, min_value, max_value, num_buckets)) }} + {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} {% endmacro %} diff --git a/macros/logger/log_info.sql b/macros/jinja_helpers/log_info.sql similarity index 61% rename from macros/logger/log_info.sql rename to macros/jinja_helpers/log_info.sql index eb655fb92ad..52b4b4a5482 100644 --- a/macros/logger/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,5 +1,5 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', packages = dbt_utils._get_utils_namespaces())(message)) }} + {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} {% endmacro %} {% macro default__log_info(message) %} diff --git a/macros/logger/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql similarity index 61% rename from macros/logger/pretty_log_format.sql rename to macros/jinja_helpers/pretty_log_format.sql index 554e274e91a..fe580d13e7d 100644 --- a/macros/logger/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,5 +1,5 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', packages = dbt_utils._get_utils_namespaces())(message)) }} + {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} {% endmacro %} {% macro default__pretty_log_format(message) %} diff --git a/macros/logger/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql similarity index 64% rename from macros/logger/pretty_time.sql rename to macros/jinja_helpers/pretty_time.sql index 653e53b170f..bad37efe48f 100644 --- a/macros/logger/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,5 +1,5 @@ {% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', packages = dbt_utils._get_utils_namespaces())(format)) }} + {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} {% endmacro %} {% macro default__pretty_time(format='%H:%M:%S') %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql new file mode 100644 index 00000000000..1b3c7272913 --- /dev/null +++ b/macros/jinja_helpers/slugify.sql @@ -0,0 +1,12 @@ +{% macro slugify(string) %} + +{#- Lower case the string -#} +{% set string = string | lower %} +{#- Replace spaces and dashes with underscores -#} +{% set string = modules.re.sub('[ -]+', '_', string) %} +{#- Only take letters, numbers, and underscores -#} +{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} + +{{ return(string) }} + +{% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index d5ed356762b..9b43bdbbf96 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,5 +1,5 @@ {% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', packages = dbt_utils._get_utils_namespaces())(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} + {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} {% endmacro %} {% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} @@ -29,7 +29,7 @@ {%- endmacro %} {% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', packages = dbt_utils._get_utils_namespaces())(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} + {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} {% endmacro %} {% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} diff --git a/macros/schema_tests/accepted_range.sql b/macros/schema_tests/accepted_range.sql index 4c599f8a920..4ce460fdf62 100644 --- a/macros/schema_tests/accepted_range.sql +++ b/macros/schema_tests/accepted_range.sql @@ -1,23 +1,24 @@ -{% macro test_accepted_range(model, min_value = none, max_value = none, inclusive = true, where = "true") %} +{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} + {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% endtest %} -{%- set column_name = kwargs.get('column_name', kwargs.get('field')) -%} +{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} with meet_condition as( - select {{ column_name }} + select {{ column_name }} from {{ model }} - where {{ where }} ), validation_errors as ( select * from meet_condition - where + where -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + 1 = 2 {%- if min_value is not none %} -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} {%- endif %} {%- if max_value is not none %} @@ -26,7 +27,7 @@ validation_errors as ( {%- endif %} ) -select count(*) +select * from validation_errors {% endmacro %} diff --git a/macros/schema_tests/at_least_one.sql b/macros/schema_tests/at_least_one.sql index 4e553635658..ce02108fed5 100644 --- a/macros/schema_tests/at_least_one.sql +++ b/macros/schema_tests/at_least_one.sql @@ -1,12 +1,10 @@ -{% macro test_at_least_one(model) %} - {{ return(adapter.dispatch('test_at_least_one', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} - -{% macro default__test_at_least_one(model) %} +{% test at_least_one(model, column_name) %} + {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} +{% endtest %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} +{% macro default__test_at_least_one(model, column_name) %} -select count(*) +select * from ( select {# In TSQL, subquery aggregate columns need aliases #} @@ -19,4 +17,4 @@ from ( ) validation_errors -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/schema_tests/cardinality_equality.sql b/macros/schema_tests/cardinality_equality.sql index c6923685601..4ec7d87fb5f 100644 --- a/macros/schema_tests/cardinality_equality.sql +++ b/macros/schema_tests/cardinality_equality.sql @@ -1,17 +1,12 @@ -{% macro test_cardinality_equality(model, to, field) %} +{% test cardinality_equality(model, column_name, to, field) %} + {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} +{% endtest %} - {{ return(adapter.dispatch('test_cardinality_equality', packages = dbt_utils._get_utils_namespaces())(model, to, field, **kwargs)) }} +{% macro default__test_cardinality_equality(model, column_name, to, field) %} -{% endmacro %} - -{% macro default__test_cardinality_equality(model, to, field) %} - -{# T-SQL doesn't let you use numbers as aliases for columns #} +{# T-SQL does not let you use numbers as aliases for columns #} {# Thus, no "GROUP BY 1" #} -{% set column_name = kwargs.get('column_name', kwargs.get('from')) %} - - with table_a as ( select {{ column_name }}, @@ -52,7 +47,7 @@ unioned as ( from except_b ) -select count(*) +select * from unioned {% endmacro %} diff --git a/macros/schema_tests/equal_rowcount.sql b/macros/schema_tests/equal_rowcount.sql index b28beed6961..713cb12a8dc 100644 --- a/macros/schema_tests/equal_rowcount.sql +++ b/macros/schema_tests/equal_rowcount.sql @@ -1,10 +1,11 @@ -{% macro test_equal_rowcount(model) %} - {{ return(adapter.dispatch('test_equal_rowcount', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} +{% test equal_rowcount(model, compare_model) %} + {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} +{% endtest %} -{% macro default__test_equal_rowcount(model) %} +{% macro default__test_equal_rowcount(model, compare_model) %} -{% set compare_model = kwargs.get('compare_model', kwargs.get('arg')) %} +{#-- Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc = 'coalesce(diff_count, 0)') }} {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} @@ -23,14 +24,15 @@ b as ( ), final as ( - select abs( - (select count_a from a) - - (select count_b from b) - ) - as diff_count + select + count_a, + count_b, + abs(count_a - count_b) as diff_count + from a + cross join b ) -select diff_count from final +select * from final -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/schema_tests/equality.sql b/macros/schema_tests/equality.sql index b2933362998..869ec9e2f42 100644 --- a/macros/schema_tests/equality.sql +++ b/macros/schema_tests/equality.sql @@ -1,9 +1,18 @@ -{% macro test_equality(model) %} - {{ return(adapter.dispatch('test_equality', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} +{% test equality(model, compare_model, compare_columns=None) %} + {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} +{% endtest %} + +{% macro default__test_equality(model, compare_model, compare_columns=None) %} -{% macro default__test_equality(model) %} +{% set set_diff %} + count(*) + coalesce(abs( + sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - + sum(case when which_diff = 'b_minus_a' then 1 else 0 end) + ), 0) +{% endset %} +{#-- Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc = set_diff) }} {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} @@ -17,14 +26,12 @@ If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} -{%- set compare_columns = kwargs.get('compare_columns', None) -%} {%- if not compare_columns -%} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} {%- endif -%} -{% set compare_model = kwargs.get('compare_model', kwargs.get('arg')) %} {% set compare_cols_csv = compare_columns | join(', ') %} with a as ( @@ -57,23 +64,12 @@ b_minus_a as ( unioned as ( - select * from a_minus_b + select 'a_minus_b' as which_diff, * from a_minus_b union all - select * from b_minus_a - -), - -final as ( - - select (select count(*) from unioned) + - (select abs( - (select count(*) from a_minus_b) - - (select count(*) from b_minus_a) - )) - as count + select 'b_minus_a' as which_diff, * from b_minus_a ) -select count from final +select * from unioned {% endmacro %} diff --git a/macros/schema_tests/expression_is_true.sql b/macros/schema_tests/expression_is_true.sql index c8a3a722abb..611685b4189 100644 --- a/macros/schema_tests/expression_is_true.sql +++ b/macros/schema_tests/expression_is_true.sql @@ -1,33 +1,22 @@ -{% macro test_expression_is_true(model, condition='1=1') %} +{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', packages = dbt_utils._get_utils_namespaces())(model, condition, **kwargs)) }} -{% endmacro %} - -{% macro default__test_expression_is_true(model, condition) %} + {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{% endtest %} -{% set expression = kwargs.get('expression', kwargs.get('arg')) %} -{% set column_name = kwargs.get('column_name') %} +{% macro default__test_expression_is_true(model, expression, column_name, condition) %} with meet_condition as ( - select * from {{ model }} where {{ condition }} - -), -validation_errors as ( - - select - * - from meet_condition - {% if column_name is none %} - where not({{ expression }}) - {%- else %} - where not({{ column_name }} {{ expression }}) - {%- endif %} - ) -select count(*) -from validation_errors +select + * +from meet_condition +{% if column_name is none %} +where not({{ expression }}) +{%- else %} +where not({{ column_name }} {{ expression }}) +{%- endif %} {% endmacro %} diff --git a/macros/schema_tests/fewer_rows_than.sql b/macros/schema_tests/fewer_rows_than.sql index 328c0ea0a75..450148b7dfa 100644 --- a/macros/schema_tests/fewer_rows_than.sql +++ b/macros/schema_tests/fewer_rows_than.sql @@ -1,36 +1,38 @@ -{% macro test_fewer_rows_than(model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', packages = dbt_utils._get_utils_namespaces())(model, combination_of_columns, quote_columns, where)) }} -{% endmacro %} +{% test fewer_rows_than(model, compare_model) %} + {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} +{% endtest %} -{% macro default__test_fewer_rows_than(model) %} +{% macro default__test_fewer_rows_than(model, compare_model) %} -{% set compare_model = kwargs.get('compare_model', kwargs.get('arg')) %} +{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} with a as ( - select count(*) as count_ourmodel from {{ model }} + select count(*) as count_our_model from {{ model }} ), b as ( - select count(*) as count_comparisonmodel from {{ compare_model }} + select count(*) as count_comparison_model from {{ compare_model }} ), counts as ( select - (select count_ourmodel from a) as count_model_with_fewer_rows, - (select count_comparisonmodel from b) as count_model_with_more_rows + count_our_model, + count_comparison_model + from a + cross join b ), final as ( - select + select *, case -- fail the test if we have more rows than the reference model and return the row count delta - when count_model_with_fewer_rows > count_model_with_more_rows then (count_model_with_fewer_rows - count_model_with_more_rows) + when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) -- fail the test if they are the same number - when count_model = count_comparison then 1 + when count_our_model = count_comparison_model then 1 -- pass the test if the delta is positive (i.e. return the number 0) else 0 end as row_count_delta @@ -38,6 +40,6 @@ final as ( ) -select row_count_delta from final +select * from final {% endmacro %} diff --git a/macros/schema_tests/mutually_exclusive_ranges.sql b/macros/schema_tests/mutually_exclusive_ranges.sql index 278a048ceb7..f2da3b952d2 100644 --- a/macros/schema_tests/mutually_exclusive_ranges.sql +++ b/macros/schema_tests/mutually_exclusive_ranges.sql @@ -1,6 +1,6 @@ -{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', packages = dbt_utils._get_utils_namespaces())(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} -{% endmacro %} +{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} + {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% endtest %} {% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} {% if gaps == 'not_allowed' %} @@ -93,5 +93,5 @@ validation_errors as ( ) ) -select count(*) from validation_errors +select * from validation_errors {% endmacro %} diff --git a/macros/schema_tests/not_accepted_values.sql b/macros/schema_tests/not_accepted_values.sql index fa2f4d83722..ab24188f8d1 100644 --- a/macros/schema_tests/not_accepted_values.sql +++ b/macros/schema_tests/not_accepted_values.sql @@ -1,8 +1,8 @@ -{% macro test_not_accepted_values(model, values) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('field')) %} -{% set quote_values = kwargs.get('quote', True) %} +{% test not_accepted_values(model, column_name, values, quote=True) %} + {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} +{% endtest %} +{% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} with all_values as ( select distinct @@ -20,7 +20,7 @@ validation_errors as ( from all_values where value_field in ( {% for value in values -%} - {% if quote_values -%} + {% if quote -%} '{{ value }}' {%- else -%} {{ value }} @@ -31,7 +31,7 @@ validation_errors as ( ) -select count(*) as validation_errors +select * from validation_errors {% endmacro %} diff --git a/macros/schema_tests/not_constant.sql b/macros/schema_tests/not_constant.sql index 19aee6499dd..781ed7d94f6 100644 --- a/macros/schema_tests/not_constant.sql +++ b/macros/schema_tests/not_constant.sql @@ -1,26 +1,19 @@ -{% macro test_not_constant(model) %} - {{ return(adapter.dispatch('test_not_constant', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} - -{% macro default__test_not_constant(model) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} - -select count(*) +{% test not_constant(model, column_name) %} + {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} +{% endtest %} -from ( +{% macro default__test_not_constant(model, column_name) %} - select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column - from {{ model }} +select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count(distinct {{ column_name }}) as filler_column - having count(distinct {{ column_name }}) = 1 +from {{ model }} - ) validation_errors +having count(distinct {{ column_name }}) = 1 {% endmacro %} diff --git a/macros/schema_tests/recency.sql b/macros/schema_tests/recency.sql index cc17e8475c2..cb9a8de65ee 100644 --- a/macros/schema_tests/recency.sql +++ b/macros/schema_tests/recency.sql @@ -1,17 +1,24 @@ -{% macro test_recency(model, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', packages = dbt_utils._get_utils_namespaces())(model, datepart, interval, **kwargs)) }} -{% endmacro %} +{% test recency(model, field, datepart, interval) %} + {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} +{% endtest %} + +{% macro default__test_recency(model, field, datepart, interval) %} + +{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} -{% macro default__test_recency(model, datepart, interval) %} +with recency as ( -{% set column_name = kwargs.get('column_name', kwargs.get('field')) %} + select max({{field}}) as most_recent + from {{ model }} + +) select - case when count(*) > 0 then 0 - else 1 - end as error_result -from {{model}} -where {{column_name}} >= - {{dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp())}} + + most_recent, + {{ threshold }} as threshold + +from recency +where most_recent < {{ threshold }} {% endmacro %} diff --git a/macros/schema_tests/relationships_where.sql b/macros/schema_tests/relationships_where.sql index 000718bda1c..c35a380a8d4 100644 --- a/macros/schema_tests/relationships_where.sql +++ b/macros/schema_tests/relationships_where.sql @@ -1,14 +1,11 @@ -{% macro test_relationships_where(model, to, field) %} - {{ return(adapter.dispatch('test_relationships_where', packages = dbt_utils._get_utils_namespaces())(model, to, field, **kwargs)) }} -{% endmacro %} +{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} + {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% endtest %} -{% macro default__test_relationships_where(model, to, field) %} +{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} -{% set column_name = kwargs.get('column_name', kwargs.get('from')) %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} -{% set from_condition = kwargs.get('from_condition', "1=1") %} -{% set to_condition = kwargs.get('to_condition', "1=1") %} with left_table as ( @@ -49,6 +46,6 @@ exceptions as ( ) -select count(*) from exceptions +select * from exceptions {% endmacro %} diff --git a/macros/schema_tests/sequential_values.sql b/macros/schema_tests/sequential_values.sql index 0af87ff39a7..83eee0c0574 100644 --- a/macros/schema_tests/sequential_values.sql +++ b/macros/schema_tests/sequential_values.sql @@ -1,10 +1,10 @@ -{% macro test_sequential_values(model, column_name, interval=1, datepart=None) %} +{% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', packages=dbt_utils._get_utils_namespaces())(model, column_name, interval, datepart, **kwargs)) }} + {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} -{% endmacro %} +{% endtest %} -{% macro default__test_sequential_values(model, column_name, interval, datepart) %} +{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} with windowed as ( @@ -27,8 +27,7 @@ validation_errors as ( {% endif %} ) -select - count(*) +select * from validation_errors {% endmacro %} diff --git a/macros/schema_tests/test_not_null_where.sql b/macros/schema_tests/test_not_null_where.sql index 9f1684ba6ed..d5dbf6c54b6 100644 --- a/macros/schema_tests/test_not_null_where.sql +++ b/macros/schema_tests/test_not_null_where.sql @@ -1,15 +1,12 @@ -{% macro test_not_null_where(model) %} - {{ return(adapter.dispatch('test_not_null_where', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} - -{% macro default__test_not_null_where(model) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set where = kwargs.get('where', kwargs.get('arg')) %} - -select count(*) -from {{ model }} -where {{ column_name }} is null -{% if where %} and {{ where }} {% endif %} - +{% test not_null_where(model, column_name) %} + {%- set deprecation_warning = ' + Warning: `dbt_utils.not_null_where` is no longer supported. + Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. + ' -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} + {{ return(test_not_null(model, column_name)) }} {% endmacro %} diff --git a/macros/schema_tests/test_unique_where.sql b/macros/schema_tests/test_unique_where.sql index 06b8e133749..e752d7b23a5 100644 --- a/macros/schema_tests/test_unique_where.sql +++ b/macros/schema_tests/test_unique_where.sql @@ -1,24 +1,12 @@ -{% macro test_unique_where(model) %} - {{ return(adapter.dispatch('test_unique_where', packages = dbt_utils._get_utils_namespaces())(model, **kwargs)) }} -{% endmacro %} - -{% macro default__test_unique_where(model) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set where = kwargs.get('where', kwargs.get('arg')) %} - -select count(*) -from ( - - select - {{ column_name }} - - from {{ model }} - where {{ column_name }} is not null - {% if where %} and {{ where }} {% endif %} - group by {{ column_name }} - having count(*) > 1 - -) validation_errors - +{% test unique_where(model, column_name) %} + {%- set deprecation_warning = ' + Warning: `dbt_utils.unique_where` is no longer supported. + Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. + ' -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} + {{ return(test_unique(model, column_name)) }} {% endmacro %} diff --git a/macros/schema_tests/unique_combination_of_columns.sql b/macros/schema_tests/unique_combination_of_columns.sql index 835d37b47d7..74ccf5c6332 100644 --- a/macros/schema_tests/unique_combination_of_columns.sql +++ b/macros/schema_tests/unique_combination_of_columns.sql @@ -1,16 +1,14 @@ -{% macro test_unique_combination_of_columns(model, quote_columns = false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', packages = dbt_utils._get_utils_namespaces())(model, quote_columns, **kwargs)) }} -{% endmacro %} - -{% macro default__test_unique_combination_of_columns(model, quote_columns = false) %} +{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} + {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% endtest %} -{%- set columns = kwargs.get('combination_of_columns', kwargs.get('arg')) %} +{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} {% if not quote_columns %} - {%- set column_list=columns %} + {%- set column_list=combination_of_columns %} {% elif quote_columns %} {%- set column_list=[] %} - {% for column in columns -%} + {% for column in combination_of_columns -%} {% set column_list = column_list.append( adapter.quote(column) ) %} {%- endfor %} {% else %} @@ -27,13 +25,12 @@ with validation_errors as ( select {{ columns_csv }} from {{ model }} - group by {{ columns_csv }} having count(*) > 1 ) -select count(*) +select * from validation_errors diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 63079f63470..6998bd2e71a 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,5 +1,5 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', packages = dbt_utils._get_utils_namespaces())(start_date, end_date, datepart)) }} + {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} @@ -24,7 +24,7 @@ {% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', packages = dbt_utils._get_utils_namespaces())(datepart, start_date, end_date)) }} + {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index e0bd9a113e8..54dcfad7dc9 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,5 +1,5 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', packages = dbt_utils._get_utils_namespaces())(upper_bound)) }} + {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} @@ -16,7 +16,7 @@ {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', packages = dbt_utils._get_utils_namespaces())(upper_bound)) }} + {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 39aa96c9b31..ee457de0ab0 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,26 +1,13 @@ -{# -This macro fetches the unique values for `column` in the table `table` - -Arguments: - table: A model `ref`, or a schema.table string for the table to query (Required) - column: The column to query for unique values - max_records: If provided, the maximum number of unique records to return (default: none) - -Returns: - A list of distinct values for the specified columns -#} - -{% macro get_column_values(table, column, max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', packages = dbt_utils._get_utils_namespaces())(table, column, max_records, default)) }} +{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} + {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} {% endmacro %} -{% macro default__get_column_values(table, column, max_records=none, default=none) -%} +{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} {{ return('') }} {% endif %} -{#-- #} {%- set target_relation = adapter.get_relation(database=table.database, schema=table.schema, @@ -40,12 +27,13 @@ Returns: {%- else -%} + select {{ column }} as value from {{ target_relation }} group by 1 - order by count(*) desc + order by {{ order_by }} {% if max_records is not none %} limit {{ max_records }} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index c8d385dfed8..6548f2dd401 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,5 +1,5 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', packages = dbt_utils._get_utils_namespaces())(query)) }} + {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index eb365ac9d2b..9325a883a8e 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,5 +1,5 @@ {% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', packages = dbt_utils._get_utils_namespaces())(schema_pattern, table_pattern, exclude, database)) }} + {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} {% endmacro %} {% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index c4e7146fa75..b6733c4baef 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,5 +1,5 @@ {% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', packages = dbt_utils._get_utils_namespaces())(schema, prefix, exclude, database)) }} + {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} {% endmacro %} {% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index d06291a96db..9185875c01f 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,5 +1,5 @@ {% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', packages = dbt_utils._get_utils_namespaces()) + {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') (schema_pattern, table_pattern, exclude, database)) }} {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index 9b1d8ef52f5..f8fdfba66aa 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,5 +1,5 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces())(schema, prefix, exclude, database)) }} + {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} {% endmacro %} {% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 10350512115..68a68cf20e7 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,5 +1,5 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', packages = dbt_utils._get_utils_namespaces())(n)) }} + {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} {% endmacro %} {%- macro default__group_by(n) -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 38fb869e439..f4d541465e1 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -11,7 +11,7 @@ The arguments should be float type. {%- endmacro %} {% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', packages = dbt_utils._get_utils_namespaces())(lat1,lon1,lat2,lon2,unit)) }} + {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} {% endmacro %} {% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 6ef9fa6c8ad..509d24f73ef 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,5 +1,5 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', packages = dbt_utils._get_utils_namespaces())(cols)) }} + {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} {% endmacro %} {% macro default__nullcheck(cols) %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index 25b30ce75d8..c9ab5838ed4 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,5 +1,5 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', packages = dbt_utils._get_utils_namespaces())(relation)) }} + {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 231a9908500..b8f12ee5606 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -51,7 +51,7 @@ Arguments: else_value=0, quote_identifiers=True, distinct=False) %} - {{ return(adapter.dispatch('pivot', packages = dbt_utils._get_utils_namespaces())(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} + {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} {% endmacro %} {% macro default__pivot(column, @@ -78,7 +78,7 @@ Arguments: {% if quote_identifiers %} as {{ adapter.quote(prefix ~ v ~ suffix) }} {% else %} - as {{prefix ~ v ~ suffix }} + as {{ dbt_utils.slugify(prefix ~ v ~ suffix) }} {% endif %} {% endif %} {% if not loop.last %},{% endif %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 8c3d39ef571..3b6195c87ac 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -2,7 +2,7 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', packages = dbt_utils._get_utils_namespaces())(*varargs)) }} + {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} {% endmacro %} {%- macro default__safe_add() -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 07cb4ebd6ed..8f82f3ae74c 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,5 +1,5 @@ {% macro star(from, relation_alias=False, except=[]) -%} - {{ return(adapter.dispatch('star', packages = dbt_utils._get_utils_namespaces())(from, relation_alias, except)) }} + {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except)) }} {% endmacro %} {% macro default__star(from, relation_alias=False, except=[]) -%} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index 7dc142d8a4b..ba5c3e446b2 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -2,7 +2,7 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', packages = dbt_utils._get_utils_namespaces())(field_list, *varargs)) }} + {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 85522847903..13bafe31afa 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,5 +1,5 @@ {%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', packages = dbt_utils._get_utils_namespaces())(relations, column_override, include, exclude, source_column_name)) }} + {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} {% endmacro %} {%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 5cabf8c9de2..8821aad284e 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -13,7 +13,7 @@ Arguments: #} {% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', packages = dbt_utils._get_utils_namespaces())(relation, cast_to, exclude, remove, field_name, value_name, table)) }} + {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} {% endmacro %} {% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index f1b0ae41e25..b1e8e4de667 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,5 +1,5 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', packages = dbt_utils._get_utils_namespaces())(field)) }} + {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} {% endmacro %} {% macro default__get_url_host(field) -%} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index d925a03d851..f363fcbb271 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,5 +1,5 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', packages = dbt_utils._get_utils_namespaces())(field, url_parameter)) }} + {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 13a6cd59483..4d2242770c1 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,5 +1,5 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', packages = dbt_utils._get_utils_namespaces())(field)) }} + {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} {% endmacro %} {% macro default__get_url_path(field) -%} diff --git a/run_test.sh b/run_test.sh index 80089ebf564..6f630e94e58 100755 --- a/run_test.sh +++ b/run_test.sh @@ -6,7 +6,7 @@ if [[ ! -f $VENV ]]; then . $VENV pip install --upgrade pip setuptools - pip install --pre "dbt<0.20.0" + pip install --pre "dbt-$1" fi . $VENV