From 65ca3b7aaee0bf7d3e0ac933c7d02fab95838a4d Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 13:48:47 +0300 Subject: [PATCH 1/6] Initial commit, adding liquid_clustered_by in adapter macros Signed-off-by: Ammar Chalifah --- dbt/adapters/databricks/impl.py | 1 + dbt/include/databricks/macros/adapters.sql | 20 ++++++++++++++++++++ dbt/include/databricks/macros/python.sql | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 7cff8f74..8f44c3fc 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -66,6 +66,7 @@ class DatabricksConfig(AdapterConfig): location_root: Optional[str] = None partition_by: Optional[Union[List[str], str]] = None clustered_by: Optional[Union[List[str], str]] = None + liquid_clustered_by: Optional[Union[List[str], str]] = None buckets: Optional[int] = None options: Optional[Dict[str, str]] = None merge_update_columns: Optional[str] = None diff --git a/dbt/include/databricks/macros/adapters.sql b/dbt/include/databricks/macros/adapters.sql index bf742c80..82bc94fe 100644 --- a/dbt/include/databricks/macros/adapters.sql +++ b/dbt/include/databricks/macros/adapters.sql @@ -43,6 +43,25 @@ {%- endif %} {%- endmacro -%} +{% macro liquid_clustered_cols(label, required=false) -%} + {{ return(adapter.dispatch('liquid_clustered_cols', 'dbt')(label, required)) }} +{%- endmacro -%} + +{% macro databricks__liquid_clustered_cols(label, required=false) -%} + {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%} + {%- if cols is not none %} + {%- if cols is string -%} + {%- set cols = [cols] -%} + {%- endif -%} + {{ label }} ( + {%- for item in cols -%} + {{ item }} + {%- if not loop.last -%},{%- endif -%} + {%- endfor -%} + ) + {%- endif %} +{%- endmacro -%} + {% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if language == 'sql' -%} @@ -62,6 +81,7 @@ {{ file_format_clause() }} {{ options_clause() }} {{ partition_cols(label="partitioned by") }} + {{ liquid_clustered_cols(label="cluster by") }} {{ clustered_cols(label="clustered by") }} {{ location_clause() }} {{ comment_clause() }} diff --git a/dbt/include/databricks/macros/python.sql b/dbt/include/databricks/macros/python.sql index bdf3741c..40dde11c 100644 --- a/dbt/include/databricks/macros/python.sql +++ b/dbt/include/databricks/macros/python.sql @@ -47,6 +47,7 @@ writer.saveAsTable("{{ target_relation }}") {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%} {%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%} {%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%} +{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%} {%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%} {%- set buckets = config.get('buckets', validator=validation.any[int]) -%} .format("{{ file_format }}") @@ -60,6 +61,9 @@ writer.saveAsTable("{{ target_relation }}") {%- endif %} .partitionBy({{ partition_by }}) {%- endif -%} +{%- if liquid_clustered_by is not none -%} + {% log("`liquid_clustered_by` is only supported in `sql` language. Ignoring liquid clustering...", True) %} +{%- endif -%} {%- if (clustered_by is not none) and (buckets is not none) -%} {%- if clustered_by is string -%} {%- set clustered_by = [clustered_by] -%} From e2f4b39b9f5876d885ceeb3fa250a59558281e61 Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 14:10:23 +0300 Subject: [PATCH 2/6] Add unit test Signed-off-by: Ammar Chalifah --- tests/unit/macros/test_adapters_macros.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/macros/test_adapters_macros.py b/tests/unit/macros/test_adapters_macros.py index 4cad1013..b4ec6a94 100644 --- a/tests/unit/macros/test_adapters_macros.py +++ b/tests/unit/macros/test_adapters_macros.py @@ -105,6 +105,27 @@ def test_macros_create_table_as_clusters(self): "using delta clustered by (cluster_1,cluster_2) into 1 buckets as select 1", ) + def test_macros_create_table_as_liquid_cluster(self): + self.config["liquid_clustered_by"] = "cluster_1" + sql = self._render_create_table_as() + + self.assertEqual( + sql, + "create or replace table my_table " + "using delta cluster by (cluster_1) as select 1", + ) + + def test_macros_create_table_as_liquid_clusters(self): + self.config["liquid_clustered_by"] = ["cluster_1", "cluster_2"] + self.config["buckets"] = "1" + sql = self._render_create_table_as() + + self.assertEqual( + sql, + "create or replace table my_table " + "using delta cluster by (cluster_1,cluster_2) as select 1", + ) + def test_macros_create_table_as_location(self): self.config["location_root"] = "/mnt/root" sql = self._render_create_table_as() @@ -140,6 +161,7 @@ def test_macros_create_table_as_tblproperties(self): def test_macros_create_table_as_all_delta(self): self.config["location_root"] = "/mnt/root" self.config["partition_by"] = ["partition_1", "partition_2"] + self.config["liquid_clustered_by"] = ["cluster_1", "cluster_2"] self.config["clustered_by"] = ["cluster_1", "cluster_2"] self.config["buckets"] = "1" self.config["persist_docs"] = {"relation": True} @@ -154,6 +176,7 @@ def test_macros_create_table_as_all_delta(self): "create or replace table my_table " "using delta " "partitioned by (partition_1,partition_2) " + "cluster by (cluster_1,cluster_2)" "clustered by (cluster_1,cluster_2) into 1 buckets " "location '/mnt/root/my_table' " "comment 'Description Test' " From e7bb541b57149924c052f6c64a505457fd52e5d1 Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 14:21:07 +0300 Subject: [PATCH 3/6] Exclude logging for liquid clustering in python language Signed-off-by: Ammar Chalifah --- dbt/include/databricks/macros/python.sql | 4 --- tests/unit/macros/test_adapters_macros.py | 42 +++++++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/dbt/include/databricks/macros/python.sql b/dbt/include/databricks/macros/python.sql index 40dde11c..bdf3741c 100644 --- a/dbt/include/databricks/macros/python.sql +++ b/dbt/include/databricks/macros/python.sql @@ -47,7 +47,6 @@ writer.saveAsTable("{{ target_relation }}") {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%} {%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%} {%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%} -{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%} {%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%} {%- set buckets = config.get('buckets', validator=validation.any[int]) -%} .format("{{ file_format }}") @@ -61,9 +60,6 @@ writer.saveAsTable("{{ target_relation }}") {%- endif %} .partitionBy({{ partition_by }}) {%- endif -%} -{%- if liquid_clustered_by is not none -%} - {% log("`liquid_clustered_by` is only supported in `sql` language. Ignoring liquid clustering...", True) %} -{%- endif -%} {%- if (clustered_by is not none) and (buckets is not none) -%} {%- if clustered_by is string -%} {%- set clustered_by = [clustered_by] -%} diff --git a/tests/unit/macros/test_adapters_macros.py b/tests/unit/macros/test_adapters_macros.py index b4ec6a94..e8569dcf 100644 --- a/tests/unit/macros/test_adapters_macros.py +++ b/tests/unit/macros/test_adapters_macros.py @@ -8,7 +8,9 @@ def setUp(self): super().setUp() self.template = self._get_template("adapters.sql") - def _render_create_table_as(self, relation="my_table", temporary=False, sql="select 1"): + def _render_create_table_as( + self, relation="my_table", temporary=False, sql="select 1" + ): self.default_context["model"].alias = relation return self._run_macro("databricks__create_table_as", temporary, relation, sql) @@ -18,7 +20,9 @@ class TestSparkMacros(TestAdaptersMacros): def test_macros_create_table_as(self): sql = self._render_create_table_as() - self.assertEqual(sql, "create or replace table my_table using delta as select 1") + self.assertEqual( + sql, "create or replace table my_table using delta as select 1" + ) def test_macros_create_table_as_file_format(self): for format in ["parquet", "hudi"]: @@ -176,7 +180,7 @@ def test_macros_create_table_as_all_delta(self): "create or replace table my_table " "using delta " "partitioned by (partition_1,partition_2) " - "cluster by (cluster_1,cluster_2)" + "cluster by (cluster_1,cluster_2) " "clustered by (cluster_1,cluster_2) into 1 buckets " "location '/mnt/root/my_table' " "comment 'Description Test' " @@ -258,7 +262,11 @@ def test_macros_get_optimize_sql(self): self.assertEqual( sql, - ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo)"), + ( + "optimize " + "`some_database`.`some_schema`.`some_table` " + "zorder by (foo)" + ), ) def test_macro_get_optimize_sql_multiple_args(self): @@ -267,7 +275,11 @@ def test_macro_get_optimize_sql_multiple_args(self): self.assertEqual( sql, - ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo, bar)"), + ( + "optimize " + "`some_database`.`some_schema`.`some_table` " + "zorder by (foo, bar)" + ), ) def test_macros_optimize_with_extraneous_info(self): @@ -301,7 +313,9 @@ def test_macros_databricks_constraints_to_dbt(self): constraint = {"name": "name", "condition": "id > 0"} r = self.__render_constraints([constraint]) - self.assertEquals(r, "[{'name': 'name', 'type': 'check', 'expression': 'id > 0'}]") + self.assertEquals( + r, "[{'name': 'name', 'type': 'check', 'expression': 'id > 0'}]" + ) def test_macros_databricks_constraints_missing_name(self): constraint = {"condition": "id > 0"} @@ -319,7 +333,9 @@ def test_macros_databricks_constraints_with_type(self): constraint = {"type": "check", "name": "name", "expression": "id > 0"} r = self.__render_constraints([constraint]) - self.assertEquals(r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]") + self.assertEquals( + r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]" + ) def test_macros_databricks_constraints_with_column_missing_expression(self): column = {"name": "col"} @@ -332,7 +348,9 @@ def test_macros_databricks_constraints_with_column_and_expression(self): constraint = {"type": "check", "name": "name", "expression": "id > 0"} r = self.__render_constraints([constraint], column) - self.assertEquals(r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]") + self.assertEquals( + r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]" + ) def test_macros_databricks_constraints_with_column_not_null(self): column = {"name": "col"} @@ -456,14 +474,18 @@ def __model(self): def test_macros_get_constraint_sql_not_null_with_columns(self): model = self.__model() - r = self.__render_constraint_sql({"type": "not_null", "columns": ["id", "name"]}, model) + r = self.__render_constraint_sql( + {"type": "not_null", "columns": ["id", "name"]}, model + ) expected = "['alter table `some_database`.`some_schema`.`some_table` change column id set not null ;', 'alter table `some_database`.`some_schema`.`some_table` change column name set not null ;']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_not_null_with_column(self): model = self.__model() - r = self.__render_constraint_sql({"type": "not_null"}, model, model["columns"]["id"]) + r = self.__render_constraint_sql( + {"type": "not_null"}, model, model["columns"]["id"] + ) expected = "['alter table `some_database`.`some_schema`.`some_table` change column id set not null ;']" # noqa: E501 assert expected in r From 98efe02a26b5326cabf6806c24ecfd1584a2c7a7 Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 14:35:56 +0300 Subject: [PATCH 4/6] Lint Signed-off-by: Ammar Chalifah --- tests/unit/macros/test_adapters_macros.py | 44 ++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/tests/unit/macros/test_adapters_macros.py b/tests/unit/macros/test_adapters_macros.py index e8569dcf..8e1175a9 100644 --- a/tests/unit/macros/test_adapters_macros.py +++ b/tests/unit/macros/test_adapters_macros.py @@ -477,7 +477,9 @@ def test_macros_get_constraint_sql_not_null_with_columns(self): r = self.__render_constraint_sql( {"type": "not_null", "columns": ["id", "name"]}, model ) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column id set not null ;', 'alter table `some_database`.`some_schema`.`some_table` change column name set not null ;']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` change column id " \ + "set not null ;', 'alter table `some_database`.`some_schema`.`some_table` " \ + "change column name set not null ;']" # noqa: E501 assert expected in r @@ -487,7 +489,8 @@ def test_macros_get_constraint_sql_not_null_with_column(self): {"type": "not_null"}, model, model["columns"]["id"] ) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column id set not null ;']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` change column id " \ + "set not null ;']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_not_null_mismatched_columns(self): @@ -496,7 +499,8 @@ def test_macros_get_constraint_sql_not_null_mismatched_columns(self): {"type": "not_null", "columns": ["name"]}, model, model["columns"]["id"] ) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column name set not null ;']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` change column name " \ + "set not null ;']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check(self): @@ -509,7 +513,8 @@ def test_macros_get_constraint_sql_check(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint check (id != name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint check (id != name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_named_constraint(self): @@ -521,7 +526,8 @@ def test_macros_get_constraint_sql_check_named_constraint(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint check (id != name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint check (id != name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_none_constraint(self): @@ -532,7 +538,8 @@ def test_macros_get_constraint_sql_check_none_constraint(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint None check (id != name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint None " \ + "check (id != name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_missing_expression(self): @@ -554,7 +561,8 @@ def test_macros_get_constraint_sql_primary_key(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint primary key(name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint primary key(name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_primary_key_with_specified_column(self): @@ -567,7 +575,8 @@ def test_macros_get_constraint_sql_primary_key_with_specified_column(self): column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint primary key(name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint primary key(name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_primary_key_with_name(self): @@ -579,7 +588,8 @@ def test_macros_get_constraint_sql_primary_key_with_name(self): column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint primary key(id);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint primary key(id);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key(self): @@ -592,7 +602,9 @@ def test_macros_get_constraint_sql_foreign_key(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint foreign key(name) references some_schema.parent_table;']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add " \ + "constraint myconstraint foreign key(name) references " \ + "some_schema.parent_table;']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_parent_column(self): @@ -606,7 +618,9 @@ def test_macros_get_constraint_sql_foreign_key_parent_column(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint foreign key(name) references some_schema.parent_table(parent_name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add " \ + "constraint myconstraint foreign key(name) references " \ + "some_schema.parent_table(parent_name);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_multiple_columns(self): @@ -620,7 +634,9 @@ def test_macros_get_constraint_sql_foreign_key_multiple_columns(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint foreign key(name, id) references some_schema.parent_table(parent_name, parent_id);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint foreign key(name, id) " \ + "references some_schema.parent_table(parent_name, parent_id);']" # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_columns_supplied_separately(self): @@ -634,5 +650,7 @@ def test_macros_get_constraint_sql_foreign_key_columns_supplied_separately(self) column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint myconstraint foreign key(id) references some_schema.parent_table(parent_name);']" # noqa: E501 + expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ + "myconstraint foreign key(id) references " \ + "some_schema.parent_table(parent_name);']" # noqa: E501 assert expected in r From d2d43f98306d2904de0d691e60ab91cb6a5a675e Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 17:52:14 +0300 Subject: [PATCH 5/6] Lint black Signed-off-by: Ammar Chalifah --- tests/unit/macros/test_adapters_macros.py | 131 +++++++++++----------- 1 file changed, 67 insertions(+), 64 deletions(-) diff --git a/tests/unit/macros/test_adapters_macros.py b/tests/unit/macros/test_adapters_macros.py index 8e1175a9..a092fdec 100644 --- a/tests/unit/macros/test_adapters_macros.py +++ b/tests/unit/macros/test_adapters_macros.py @@ -8,9 +8,7 @@ def setUp(self): super().setUp() self.template = self._get_template("adapters.sql") - def _render_create_table_as( - self, relation="my_table", temporary=False, sql="select 1" - ): + def _render_create_table_as(self, relation="my_table", temporary=False, sql="select 1"): self.default_context["model"].alias = relation return self._run_macro("databricks__create_table_as", temporary, relation, sql) @@ -20,9 +18,7 @@ class TestSparkMacros(TestAdaptersMacros): def test_macros_create_table_as(self): sql = self._render_create_table_as() - self.assertEqual( - sql, "create or replace table my_table using delta as select 1" - ) + self.assertEqual(sql, "create or replace table my_table using delta as select 1") def test_macros_create_table_as_file_format(self): for format in ["parquet", "hudi"]: @@ -115,8 +111,7 @@ def test_macros_create_table_as_liquid_cluster(self): self.assertEqual( sql, - "create or replace table my_table " - "using delta cluster by (cluster_1) as select 1", + "create or replace table my_table " "using delta cluster by (cluster_1) as select 1", ) def test_macros_create_table_as_liquid_clusters(self): @@ -262,11 +257,7 @@ def test_macros_get_optimize_sql(self): self.assertEqual( sql, - ( - "optimize " - "`some_database`.`some_schema`.`some_table` " - "zorder by (foo)" - ), + ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo)"), ) def test_macro_get_optimize_sql_multiple_args(self): @@ -275,11 +266,7 @@ def test_macro_get_optimize_sql_multiple_args(self): self.assertEqual( sql, - ( - "optimize " - "`some_database`.`some_schema`.`some_table` " - "zorder by (foo, bar)" - ), + ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo, bar)"), ) def test_macros_optimize_with_extraneous_info(self): @@ -313,9 +300,7 @@ def test_macros_databricks_constraints_to_dbt(self): constraint = {"name": "name", "condition": "id > 0"} r = self.__render_constraints([constraint]) - self.assertEquals( - r, "[{'name': 'name', 'type': 'check', 'expression': 'id > 0'}]" - ) + self.assertEquals(r, "[{'name': 'name', 'type': 'check', 'expression': 'id > 0'}]") def test_macros_databricks_constraints_missing_name(self): constraint = {"condition": "id > 0"} @@ -333,9 +318,7 @@ def test_macros_databricks_constraints_with_type(self): constraint = {"type": "check", "name": "name", "expression": "id > 0"} r = self.__render_constraints([constraint]) - self.assertEquals( - r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]" - ) + self.assertEquals(r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]") def test_macros_databricks_constraints_with_column_missing_expression(self): column = {"name": "col"} @@ -348,9 +331,7 @@ def test_macros_databricks_constraints_with_column_and_expression(self): constraint = {"type": "check", "name": "name", "expression": "id > 0"} r = self.__render_constraints([constraint], column) - self.assertEquals( - r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]" - ) + self.assertEquals(r, "[{'type': 'check', 'name': 'name', 'expression': 'id > 0'}]") def test_macros_databricks_constraints_with_column_not_null(self): column = {"name": "col"} @@ -474,23 +455,23 @@ def __model(self): def test_macros_get_constraint_sql_not_null_with_columns(self): model = self.__model() - r = self.__render_constraint_sql( - {"type": "not_null", "columns": ["id", "name"]}, model - ) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column id " \ - "set not null ;', 'alter table `some_database`.`some_schema`.`some_table` " \ - "change column name set not null ;']" # noqa: E501 + r = self.__render_constraint_sql({"type": "not_null", "columns": ["id", "name"]}, model) + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` change column id " + "set not null ;', 'alter table `some_database`.`some_schema`.`some_table` " + "change column name set not null ;']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_not_null_with_column(self): model = self.__model() - r = self.__render_constraint_sql( - {"type": "not_null"}, model, model["columns"]["id"] - ) + r = self.__render_constraint_sql({"type": "not_null"}, model, model["columns"]["id"]) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column id " \ - "set not null ;']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` change column id " + "set not null ;']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_not_null_mismatched_columns(self): @@ -499,8 +480,10 @@ def test_macros_get_constraint_sql_not_null_mismatched_columns(self): {"type": "not_null", "columns": ["name"]}, model, model["columns"]["id"] ) - expected = "['alter table `some_database`.`some_schema`.`some_table` change column name " \ - "set not null ;']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` change column name " + "set not null ;']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check(self): @@ -513,8 +496,10 @@ def test_macros_get_constraint_sql_check(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint check (id != name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint check (id != name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_named_constraint(self): @@ -526,8 +511,10 @@ def test_macros_get_constraint_sql_check_named_constraint(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint check (id != name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint check (id != name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_none_constraint(self): @@ -538,8 +525,10 @@ def test_macros_get_constraint_sql_check_none_constraint(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint None " \ - "check (id != name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint None " + "check (id != name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_check_missing_expression(self): @@ -561,8 +550,10 @@ def test_macros_get_constraint_sql_primary_key(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint primary key(name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint primary key(name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_primary_key_with_specified_column(self): @@ -575,8 +566,10 @@ def test_macros_get_constraint_sql_primary_key_with_specified_column(self): column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint primary key(name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint primary key(name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_primary_key_with_name(self): @@ -588,8 +581,10 @@ def test_macros_get_constraint_sql_primary_key_with_name(self): column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint primary key(id);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint primary key(id);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key(self): @@ -602,9 +597,11 @@ def test_macros_get_constraint_sql_foreign_key(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add " \ - "constraint myconstraint foreign key(name) references " \ - "some_schema.parent_table;']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add " + "constraint myconstraint foreign key(name) references " + "some_schema.parent_table;']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_parent_column(self): @@ -618,9 +615,11 @@ def test_macros_get_constraint_sql_foreign_key_parent_column(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add " \ - "constraint myconstraint foreign key(name) references " \ - "some_schema.parent_table(parent_name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add " + "constraint myconstraint foreign key(name) references " + "some_schema.parent_table(parent_name);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_multiple_columns(self): @@ -634,9 +633,11 @@ def test_macros_get_constraint_sql_foreign_key_multiple_columns(self): } r = self.__render_constraint_sql(constraint, model) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint foreign key(name, id) " \ - "references some_schema.parent_table(parent_name, parent_id);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint foreign key(name, id) " + "references some_schema.parent_table(parent_name, parent_id);']" + ) # noqa: E501 assert expected in r def test_macros_get_constraint_sql_foreign_key_columns_supplied_separately(self): @@ -650,7 +651,9 @@ def test_macros_get_constraint_sql_foreign_key_columns_supplied_separately(self) column = {"name": "id"} r = self.__render_constraint_sql(constraint, model, column) - expected = "['alter table `some_database`.`some_schema`.`some_table` add constraint " \ - "myconstraint foreign key(id) references " \ - "some_schema.parent_table(parent_name);']" # noqa: E501 + expected = ( + "['alter table `some_database`.`some_schema`.`some_table` add constraint " + "myconstraint foreign key(id) references " + "some_schema.parent_table(parent_name);']" + ) # noqa: E501 assert expected in r From f78c1517422138bfb50449d5bfff8adabb58f0fb Mon Sep 17 00:00:00 2001 From: Ammar Chalifah Date: Fri, 28 Jul 2023 18:29:43 +0300 Subject: [PATCH 6/6] Add changelog Signed-off-by: Ammar Chalifah --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 891bfc24..5d51ea18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - Fix issue where the show tables extended command is limited to 2048 characters. ([#326](https://github.com/databricks/dbt-databricks/pull/326)) - Extend python model support to cover the same config options as SQL ([#379](https://github.com/databricks/dbt-databricks/pull/379)) +### Features + +- Add `liquid_clustered_by` config to enable Liquid Clustering for Delta-based dbt models. + ## dbt-databricks 1.5.5 (July 7, 2023) ### Fixes