From cbc9672dcc86da79fc2545393b47d9de23a3a398 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Mon, 12 Dec 2022 17:57:09 +0100 Subject: [PATCH 01/23] Create bigquery adapter macro for constraints --- dbt/include/bigquery/macros/adapters.sql | 1 + .../macros/utils/get_columns_spec_ddl.sql | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index 07cf3c3e5..e5c298567 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -55,6 +55,7 @@ {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} {{ bigquery_table_options(config, model, temporary) }} + {{ get_columns_spec_ddl() }} as ( {{ compiled_code }} ); diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql new file mode 100644 index 000000000..80de97deb --- /dev/null +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -0,0 +1,18 @@ +{% macro bigquery__get_columns_spec_ddl() %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {% if config.get('constraints_enabled', False) %} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns -%} + {%- set col = user_provided_columns[i] -%} + {% set constraints = col['constraints'] -%} + {%- set checks = col['check'] -%} + {%- if checks -%} + {{exceptions.warn("We noticed you have `check` in your configs, these are NOT compatible with BigQuery and will be ignored")}} + {%- endif %} + {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} + {%- endfor %} + ) + {% endif %} +{% endmacro %} + From ae993884d2b7cba847f9b95b4ac672c1a4c9cfa1 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 14 Dec 2022 19:03:51 +0100 Subject: [PATCH 02/23] fix order --- dbt/include/bigquery/macros/adapters.sql | 3 ++- dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index e5c298567..620b31a77 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -52,10 +52,11 @@ {{ sql_header if sql_header is not none }} create or replace table {{ relation }} + {{ get_columns_spec_ddl() }} {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} + {{ bigquery_table_options(config, model, temporary) }} - {{ get_columns_spec_ddl() }} as ( {{ compiled_code }} ); diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql index 80de97deb..c24de5f20 100644 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -5,12 +5,12 @@ ( {% for i in user_provided_columns -%} {%- set col = user_provided_columns[i] -%} - {% set constraints = col['constraints'] -%} + {% set constraints = col['constraint'] -%} {%- set checks = col['check'] -%} {%- if checks -%} {{exceptions.warn("We noticed you have `check` in your configs, these are NOT compatible with BigQuery and will be ignored")}} {%- endif %} - {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} + {{ col['name'] }} {{ col['data_type'] }} {{ col['constraint'] or "" }} {{ "," if not loop.last }} {%- endfor %} ) {% endif %} From 5eb87b6805968c696ae1134a2fc1abc61a9deb78 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 19:01:29 +0100 Subject: [PATCH 03/23] update checks validation and constraints list --- .../bigquery/macros/utils/get_columns_spec_ddl.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql index c24de5f20..b66a6512a 100644 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -1,18 +1,18 @@ {% macro bigquery__get_columns_spec_ddl() %} {# loop through user_provided_columns to create DDL with data types and constraints #} {% if config.get('constraints_enabled', False) %} + {%- set ns = namespace(at_least_one_check=False) -%} {%- set user_provided_columns = model['columns'] -%} ( {% for i in user_provided_columns -%} {%- set col = user_provided_columns[i] -%} {% set constraints = col['constraint'] -%} - {%- set checks = col['check'] -%} - {%- if checks -%} - {{exceptions.warn("We noticed you have `check` in your configs, these are NOT compatible with BigQuery and will be ignored")}} - {%- endif %} - {{ col['name'] }} {{ col['data_type'] }} {{ col['constraint'] or "" }} {{ "," if not loop.last }} + {%- set ns.at_least_one_check = ns.at_least_one_check or col['check'] %} + {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} {%- endfor %} ) + {%- if ns.at_least_one_check -%} + {{exceptions.warn("We noticed you have `check` configs, these are NOT compatible with BigQuery and will be ignored")}} + {%- endif %} {% endif %} {% endmacro %} - From bc10cd2544bcfbb26426c4bc73717c6fff494d23 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 19:34:00 +0100 Subject: [PATCH 04/23] add test --- dev-requirements.txt | 5 +- .../constraints/test_bigquery_constraints.py | 196 ++++++++++++++++++ 2 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 tests/functional/adapter/constraints/test_bigquery_constraints.py diff --git a/dev-requirements.txt b/dev-requirements.txt index de146ae27..54b5abfee 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter - +git+https://github.com/dbt-labs/dbt-core.git@dbt-constraints#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@dbt-constraints#egg=dbt-tests-adapter&subdirectory=tests/adapter black~=22.8.0 bumpversion~=0.6.0 flake8 diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py new file mode 100644 index 000000000..6af0103dc --- /dev/null +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -0,0 +1,196 @@ +import pytest +import re +import json +from dbt.tests.util import ( + run_dbt, + get_manifest, + run_dbt_and_capture +) +from dbt.tests.adapter.constraints.test_constraints import ( + TestModelLevelConstraintsEnabledConfigs, + TestModelLevelConstraintsDisabledConfigs, + TestSchemaConstraintsEnabledConfigs, + TestModelLevelConstraintsErrorMessages +) + + +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_error_sql = """ +{{ + config( + materialized = "table" + ) +}} +select + null as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + check: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: date + - name: my_model_error + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + check: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: date +""" + +database_name = "database_placeholder" +schema_name = "schema_placeholder" + +_expected_sql = f""" + create table + "{database_name}"."{schema_name}"."my_model__dbt_tmp" + + ( + id integer not null, + color varchar, + date_day date, + primary key(id) + ) + + + + ; + insert into "{database_name}"."{schema_name}"."my_model__dbt_tmp" + ( + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day + ) + ; +""" + + +class BaseConstraintsEnabledModelvsProject: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+constraints_enabled": True, + "subdirectory": { + "+constraints_enabled": False, + }, + } + } + } + + +class TestRedshiftConstraints(BaseConstraintsEnabledModelvsProject): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_error.sql": my_model_error_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql + + def test__model_constraints_DDL(self, project, expected_sql): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + with open("./target/run/test/models/my_model.sql", "r") as fp: + generated_sql = fp.read() + + with open("./target/manifest.json", "r") as fp: + generated_manifest = json.load(fp) + + model_unique_id = 'model.test.my_model' + schema_name_generated = (generated_manifest['nodes'][model_unique_id]['schema']) + database_name_generated = (generated_manifest['nodes'][model_unique_id]['database']) + + if expected_sql: + expected_sql = expected_sql.replace(schema_name, schema_name_generated) + expected_sql = expected_sql.replace(database_name, database_name_generated) + generated_sql_check = re.sub(r"\s+", "", generated_sql).lower() + expected_sql_check = re.sub(r"\s+", "", expected_sql).lower() + assert ( + expected_sql_check == generated_sql_check + ), f"generated sql did not match expected: {generated_sql}" + + def test__rollback(self, project): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + with open("./models/my_model.sql", "r") as fp: + my_model_sql_original = fp.read() + + my_model_sql_error = my_model_sql_original.replace("1 as id", "null as id") + + with open("./models/my_model.sql", "w") as fp: + fp.write(my_model_sql_error) + + results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(results) == 1 + + with open("./target/manifest.json", "r") as fp: + generated_manifest = json.load(fp) + + model_unique_id = 'model.test.my_model' + schema_name_generated = (generated_manifest['nodes'][model_unique_id]['schema']) + database_name_generated = (generated_manifest['nodes'][model_unique_id]['database']) + + # verify the previous table exists + sql = f""" + select id from {database_name_generated}.{schema_name_generated}.my_model where id = 1 + """ + results = project.run_sql(sql, fetch="all") + assert len(results) == 1 + assert results[0][0] == 1 + + def test__constraints_enforcement(self, project): + + results, log_output = run_dbt_and_capture(['run', '-s', 'my_model_error'], expect_pass=False) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_error" + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + + assert constraints_enabled_actual_config is True + + expected_constraints_error = 'Cannot insert a NULL value into column id' + assert expected_constraints_error in log_output \ No newline at end of file From c570acd589f5aa1150f320ae3de25d169673eeb4 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 19:37:45 +0100 Subject: [PATCH 05/23] add changie --- .changes/unreleased/Features-20221220-193731.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20221220-193731.yaml diff --git a/.changes/unreleased/Features-20221220-193731.yaml b/.changes/unreleased/Features-20221220-193731.yaml new file mode 100644 index 000000000..0e9cb4262 --- /dev/null +++ b/.changes/unreleased/Features-20221220-193731.yaml @@ -0,0 +1,7 @@ +kind: Features +body: 'dbt-constraints support for BigQuery as per dbt-core issue #1358' +time: 2022-12-20T19:37:31.982821+01:00 +custom: + Author: victoriapm + Issue: "426" + PR: "426" From b731d423ab6758af8d3640ee8ec92dbae1b0617e Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 20:02:13 +0100 Subject: [PATCH 06/23] update issue number --- .changes/unreleased/Features-20221220-193731.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Features-20221220-193731.yaml b/.changes/unreleased/Features-20221220-193731.yaml index 0e9cb4262..dbd16ae0f 100644 --- a/.changes/unreleased/Features-20221220-193731.yaml +++ b/.changes/unreleased/Features-20221220-193731.yaml @@ -3,5 +3,5 @@ body: 'dbt-constraints support for BigQuery as per dbt-core issue #1358' time: 2022-12-20T19:37:31.982821+01:00 custom: Author: victoriapm - Issue: "426" + Issue: "444" PR: "426" From b548eab10ae9bea1093a689755bfa6af7d6264f0 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 20:23:15 +0100 Subject: [PATCH 07/23] fix trailing space --- dbt/include/bigquery/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index 620b31a77..bee1ab0dc 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -55,7 +55,7 @@ {{ get_columns_spec_ddl() }} {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} - + {{ bigquery_table_options(config, model, temporary) }} as ( {{ compiled_code }} From 4ff89d17aaaa2dce1f1d047ae42964ca2c73091b Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 20:52:37 +0100 Subject: [PATCH 08/23] fix test name --- .../functional/adapter/constraints/test_bigquery_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 6af0103dc..602331f22 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -117,7 +117,7 @@ def project_config_update(self): } -class TestRedshiftConstraints(BaseConstraintsEnabledModelvsProject): +class TestBigQueryConstraints(BaseConstraintsEnabledModelvsProject): @pytest.fixture(scope="class") def models(self): return { From aaea6a80a40ac5a2f9829b29141ba0976a5b4017 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 21:23:09 +0100 Subject: [PATCH 09/23] update BQ data types --- .../adapter/constraints/test_bigquery_constraints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 602331f22..27e335bc8 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -53,7 +53,7 @@ tests: - unique - name: color - data_type: varchar + data_type: string - name: date_day data_type: date - name: my_model_error @@ -68,7 +68,7 @@ tests: - unique - name: color - data_type: varchar + data_type: string - name: date_day data_type: date """ @@ -82,7 +82,7 @@ ( id integer not null, - color varchar, + color string, date_day date, primary key(id) ) From a074d7d73a9bacaa6cbc49d1e2e6c50344d43639 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 21:48:54 +0100 Subject: [PATCH 10/23] Update test_bigquery_constraints.py --- .../constraints/test_bigquery_constraints.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 27e335bc8..16625cd95 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -77,28 +77,24 @@ schema_name = "schema_placeholder" _expected_sql = f""" - create table + create or replace table "{database_name}"."{schema_name}"."my_model__dbt_tmp" ( id integer not null, color string, - date_day date, - primary key(id) + date_day date ) - - - - ; - insert into "{database_name}"."{schema_name}"."my_model__dbt_tmp" - ( + + OPTIONS() + as ( -select + select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day - ) - ; + ); + """ From 60dda354c185726085d78da840ca59065271d055 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Tue, 20 Dec 2022 22:28:15 +0100 Subject: [PATCH 11/23] Update test_bigquery_constraints.py --- .../functional/adapter/constraints/test_bigquery_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 16625cd95..30c6c9e5d 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -78,7 +78,7 @@ _expected_sql = f""" create or replace table - "{database_name}"."{schema_name}"."my_model__dbt_tmp" + "{database_name}"."{schema_name}"."my_model" ( id integer not null, From 99abbbd283893febf71e4e1f1be0b20e9850559d Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola <4315804+Victoriapm@users.noreply.github.com> Date: Wed, 21 Dec 2022 09:46:40 +0100 Subject: [PATCH 12/23] fix quotes Co-authored-by: Sung Won Chung --- .../functional/adapter/constraints/test_bigquery_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 30c6c9e5d..bc5317db3 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -78,7 +78,7 @@ _expected_sql = f""" create or replace table - "{database_name}"."{schema_name}"."my_model" + `{database_name}`.`{schema_name}`.`my_model` ( id integer not null, From 96939bfafe25ed12cfce6cebcfeca82751ebb3ed Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 21 Dec 2022 15:44:31 +0100 Subject: [PATCH 13/23] Update test_bigquery_constraints.py --- .../constraints/test_bigquery_constraints.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index bc5317db3..1329daa7d 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -48,7 +48,7 @@ - name: id data_type: integer description: hello - constraints: ['not null','primary key'] + constraints: ['not null'] check: (id > 0) tests: - unique @@ -63,7 +63,7 @@ - name: id data_type: integer description: hello - constraints: ['not null','primary key'] + constraints: ['not null'] check: (id > 0) tests: - unique @@ -77,24 +77,29 @@ schema_name = "schema_placeholder" _expected_sql = f""" - create or replace table - `{database_name}`.`{schema_name}`.`my_model` + create or replace table `{database_name}`.`{schema_name}`.`my_model` + + + + ( - ( - id integer not null, - color string, - date_day date - ) + id integer , + color string , + date_day date + ) + + + + OPTIONS() as ( - select +select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day ); - """ From e12a4c04effd525af36557cf18efc940c446e8c0 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 21 Dec 2022 18:09:53 +0100 Subject: [PATCH 14/23] fix constratints config --- .../bigquery/macros/utils/get_columns_spec_ddl.sql | 2 +- .../adapter/constraints/test_bigquery_constraints.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql index b66a6512a..5ee1001f5 100644 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -6,7 +6,7 @@ ( {% for i in user_provided_columns -%} {%- set col = user_provided_columns[i] -%} - {% set constraints = col['constraint'] -%} + {% set constraints = col['constraints'] -%} {%- set ns.at_least_one_check = ns.at_least_one_check or col['check'] %} {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} {%- endfor %} diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 1329daa7d..3a6787ba5 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -48,7 +48,8 @@ - name: id data_type: integer description: hello - constraints: ['not null'] + constraint: + - not null check: (id > 0) tests: - unique @@ -63,7 +64,8 @@ - name: id data_type: integer description: hello - constraints: ['not null'] + constraint: + - not null check: (id > 0) tests: - unique @@ -83,7 +85,7 @@ ( - id integer , + id integer not null , color string , date_day date ) From 374f135db224317ecacf78f0968b6382da20e1f6 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 21 Dec 2022 18:13:48 +0100 Subject: [PATCH 15/23] fix config --- .../adapter/constraints/test_bigquery_constraints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 3a6787ba5..bc53ae5dc 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -48,7 +48,7 @@ - name: id data_type: integer description: hello - constraint: + constraints: - not null check: (id > 0) tests: @@ -64,7 +64,7 @@ - name: id data_type: integer description: hello - constraint: + constraints: - not null check: (id > 0) tests: From e727cc889fd1c7e1bbdefb1c3724e558eaf52902 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 21 Dec 2022 18:39:29 +0100 Subject: [PATCH 16/23] Update test_bigquery_constraints.py --- .../functional/adapter/constraints/test_bigquery_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index bc53ae5dc..650d6f420 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -195,5 +195,5 @@ def test__constraints_enforcement(self, project): assert constraints_enabled_actual_config is True - expected_constraints_error = 'Cannot insert a NULL value into column id' + expected_constraints_error = 'Required field id cannot be null' assert expected_constraints_error in log_output \ No newline at end of file From 1ee39def3e9d2c11ef7e30fb6e547c11bfb03981 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola Date: Wed, 21 Dec 2022 19:04:19 +0100 Subject: [PATCH 17/23] update constraints config --- .../adapter/constraints/test_bigquery_constraints.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py index 650d6f420..364a6ea7d 100644 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ b/tests/functional/adapter/constraints/test_bigquery_constraints.py @@ -48,8 +48,7 @@ - name: id data_type: integer description: hello - constraints: - - not null + constraints: ['not null'] check: (id > 0) tests: - unique @@ -64,8 +63,7 @@ - name: id data_type: integer description: hello - constraints: - - not null + constraints: ['not null'] check: (id > 0) tests: - unique From 6c46db4542075194e7570b510f714b76f1af3dd4 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 1 Feb 2023 11:54:11 +0100 Subject: [PATCH 18/23] Switch to new functional tests --- dbt/include/bigquery/macros/adapters.sql | 1 + .../constraints/test_bigquery_constraints.py | 197 ------------------ tests/functional/adapter/test_constraints.py | 63 ++++++ 3 files changed, 64 insertions(+), 197 deletions(-) delete mode 100644 tests/functional/adapter/constraints/test_bigquery_constraints.py create mode 100644 tests/functional/adapter/test_constraints.py diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index bee1ab0dc..7d689aaeb 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -53,6 +53,7 @@ create or replace table {{ relation }} {{ get_columns_spec_ddl() }} + {{ get_assert_columns_equivalent(sql) }} {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} diff --git a/tests/functional/adapter/constraints/test_bigquery_constraints.py b/tests/functional/adapter/constraints/test_bigquery_constraints.py deleted file mode 100644 index 364a6ea7d..000000000 --- a/tests/functional/adapter/constraints/test_bigquery_constraints.py +++ /dev/null @@ -1,197 +0,0 @@ -import pytest -import re -import json -from dbt.tests.util import ( - run_dbt, - get_manifest, - run_dbt_and_capture -) -from dbt.tests.adapter.constraints.test_constraints import ( - TestModelLevelConstraintsEnabledConfigs, - TestModelLevelConstraintsDisabledConfigs, - TestSchemaConstraintsEnabledConfigs, - TestModelLevelConstraintsErrorMessages -) - - -my_model_sql = """ -{{ - config( - materialized = "table" - ) -}} -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -my_model_error_sql = """ -{{ - config( - materialized = "table" - ) -}} -select - null as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - constraints_enabled: true - columns: - - name: id - data_type: integer - description: hello - constraints: ['not null'] - check: (id > 0) - tests: - - unique - - name: color - data_type: string - - name: date_day - data_type: date - - name: my_model_error - config: - constraints_enabled: true - columns: - - name: id - data_type: integer - description: hello - constraints: ['not null'] - check: (id > 0) - tests: - - unique - - name: color - data_type: string - - name: date_day - data_type: date -""" - -database_name = "database_placeholder" -schema_name = "schema_placeholder" - -_expected_sql = f""" - create or replace table `{database_name}`.`{schema_name}`.`my_model` - - - - ( - - id integer not null , - color string , - date_day date - ) - - - - - - OPTIONS() - as ( - -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day - ); -""" - - -class BaseConstraintsEnabledModelvsProject: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+constraints_enabled": True, - "subdirectory": { - "+constraints_enabled": False, - }, - } - } - } - - -class TestBigQueryConstraints(BaseConstraintsEnabledModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "my_model_error.sql": my_model_error_sql, - "constraints_schema.yml": model_schema_yml, - } - - @pytest.fixture(scope="class") - def expected_sql(self): - return _expected_sql - - def test__model_constraints_DDL(self, project, expected_sql): - results = run_dbt(["run", "-s", "my_model"]) - assert len(results) == 1 - with open("./target/run/test/models/my_model.sql", "r") as fp: - generated_sql = fp.read() - - with open("./target/manifest.json", "r") as fp: - generated_manifest = json.load(fp) - - model_unique_id = 'model.test.my_model' - schema_name_generated = (generated_manifest['nodes'][model_unique_id]['schema']) - database_name_generated = (generated_manifest['nodes'][model_unique_id]['database']) - - if expected_sql: - expected_sql = expected_sql.replace(schema_name, schema_name_generated) - expected_sql = expected_sql.replace(database_name, database_name_generated) - generated_sql_check = re.sub(r"\s+", "", generated_sql).lower() - expected_sql_check = re.sub(r"\s+", "", expected_sql).lower() - assert ( - expected_sql_check == generated_sql_check - ), f"generated sql did not match expected: {generated_sql}" - - def test__rollback(self, project): - results = run_dbt(["run", "-s", "my_model"]) - assert len(results) == 1 - - with open("./models/my_model.sql", "r") as fp: - my_model_sql_original = fp.read() - - my_model_sql_error = my_model_sql_original.replace("1 as id", "null as id") - - with open("./models/my_model.sql", "w") as fp: - fp.write(my_model_sql_error) - - results = run_dbt(["run", "-s", "my_model"], expect_pass=False) - assert len(results) == 1 - - with open("./target/manifest.json", "r") as fp: - generated_manifest = json.load(fp) - - model_unique_id = 'model.test.my_model' - schema_name_generated = (generated_manifest['nodes'][model_unique_id]['schema']) - database_name_generated = (generated_manifest['nodes'][model_unique_id]['database']) - - # verify the previous table exists - sql = f""" - select id from {database_name_generated}.{schema_name_generated}.my_model where id = 1 - """ - results = project.run_sql(sql, fetch="all") - assert len(results) == 1 - assert results[0][0] == 1 - - def test__constraints_enforcement(self, project): - - results, log_output = run_dbt_and_capture(['run', '-s', 'my_model_error'], expect_pass=False) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model_error" - my_model_config = manifest.nodes[model_id].config - constraints_enabled_actual_config = my_model_config.constraints_enabled - - assert constraints_enabled_actual_config is True - - expected_constraints_error = 'Required field id cannot be null' - assert expected_constraints_error in log_output \ No newline at end of file diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 000000000..e024e7b55 --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,63 @@ +import pytest +from dbt.tests.util import ( + run_dbt, + get_manifest, + run_dbt_and_capture +) +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsColumnsEqual, + BaseConstraintsRuntimeEnforcement +) +from dbt.tests.adapter.constraints.fixtures import ( + my_model_sql, + my_model_wrong_order_sql, + my_model_wrong_name_sql, + model_schema_yml, +) + +_expected_sql_bigquery = """ +create or replace table `{0}`.`{1}`.`my_model` ( + id integer not null , + color string , + date_day date +) +OPTIONS() +as ( + select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +); +""" + +# Different on BigQuery: +# - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) +# - raises an explicit error, if you try to set a primary key constraint, because it's not enforced +constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "") + +class TestBigQueryConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +class TestBigQueryConstraintsRuntimeEnforcement(BaseConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constraints_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, project): + return _expected_sql_bigquery.format(project.database, project.test_schema) + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Required field id cannot be null"] From 16c00ffb57ca63582554ba757c822217edaf918a Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 1 Feb 2023 12:31:50 +0100 Subject: [PATCH 19/23] Fix failing tests --- dbt/include/bigquery/macros/adapters.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index 7d689aaeb..d80adc70c 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -52,8 +52,10 @@ {{ sql_header if sql_header is not none }} create or replace table {{ relation }} - {{ get_columns_spec_ddl() }} - {{ get_assert_columns_equivalent(sql) }} + {% if config.get('constraints_enabled', False) %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_columns_spec_ddl() }} + {% endif %} {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} From 7ad6851cef2145dd91454bae12b899b70ce2595a Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 2 Feb 2023 11:18:00 +0100 Subject: [PATCH 20/23] Small cleanup --- tests/functional/adapter/test_constraints.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index e024e7b55..b9a271b8c 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -1,9 +1,5 @@ import pytest -from dbt.tests.util import ( - run_dbt, - get_manifest, - run_dbt_and_capture -) +from dbt.tests.util import relation_from_name from dbt.tests.adapter.constraints.test_constraints import ( BaseConstraintsColumnsEqual, BaseConstraintsRuntimeEnforcement @@ -16,7 +12,7 @@ ) _expected_sql_bigquery = """ -create or replace table `{0}`.`{1}`.`my_model` ( +create or replace table {0} ( id integer not null , color string , date_day date @@ -38,7 +34,6 @@ class TestBigQueryConstraintsColumnsEqual(BaseConstraintsColumnsEqual): @pytest.fixture(scope="class") def models(self): - return { "my_model_wrong_order.sql": my_model_wrong_order_sql, "my_model_wrong_name.sql": my_model_wrong_name_sql, @@ -56,7 +51,8 @@ def models(self): @pytest.fixture(scope="class") def expected_sql(self, project): - return _expected_sql_bigquery.format(project.database, project.test_schema) + relation = relation_from_name(project.adapter, "my_model") + return _expected_sql_bigquery.format(relation) @pytest.fixture(scope="class") def expected_error_messages(self): From 4f419c55c51105a6bed605b9236976bdb9af98e1 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 15 Feb 2023 16:56:21 +0100 Subject: [PATCH 21/23] Reset to dbt-core main --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 54b5abfee..f489a04b3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@dbt-constraints#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@dbt-constraints#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter black~=22.8.0 bumpversion~=0.6.0 flake8 From 484038cb3d0661b1300db49974ff7e89be23e92f Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 16 Feb 2023 22:23:24 +0100 Subject: [PATCH 22/23] Resolve PR comment --- dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql index 5ee1001f5..6138f672b 100644 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -7,12 +7,12 @@ {% for i in user_provided_columns -%} {%- set col = user_provided_columns[i] -%} {% set constraints = col['constraints'] -%} - {%- set ns.at_least_one_check = ns.at_least_one_check or col['check'] %} + {%- set ns.at_least_one_check = ns.at_least_one_check or col['constraints_check'] %} {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} {%- endfor %} ) {%- if ns.at_least_one_check -%} - {{exceptions.warn("We noticed you have `check` configs, these are NOT compatible with BigQuery and will be ignored")}} + {{exceptions.warn("We noticed you have `constraints_check` configs, these are NOT compatible with BigQuery and will be ignored")}} {%- endif %} {% endif %} {% endmacro %} From 2ac91c938e82264ed201a33bbfe4db2d161a23a0 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 17 Feb 2023 11:33:18 +0100 Subject: [PATCH 23/23] PR feedback --- dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql index 6138f672b..4f2720b7e 100644 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -1,6 +1,5 @@ {% macro bigquery__get_columns_spec_ddl() %} {# loop through user_provided_columns to create DDL with data types and constraints #} - {% if config.get('constraints_enabled', False) %} {%- set ns = namespace(at_least_one_check=False) -%} {%- set user_provided_columns = model['columns'] -%} ( @@ -14,5 +13,4 @@ {%- if ns.at_least_one_check -%} {{exceptions.warn("We noticed you have `constraints_check` configs, these are NOT compatible with BigQuery and will be ignored")}} {%- endif %} - {% endif %} {% endmacro %}