Ensure optimize is run with liquid_clustered_by

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## dbt-databricks 1.6.x (Release TBD)
 
+### Fixes
+
+- Optimize now runs after creating / updating liquid clustering tables ([463](https://github.com/databricks/dbt-databricks/pull/463))
+
 ## dbt-databricks 1.6.5 (September 26, 2023)
 
 ### Features

dbt/include/databricks/macros/adapters.sql

@@ -368,32 +368,38 @@
   {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}
 {% endmacro %}
 
-{% macro databricks__optimize(relation) %}
-  {% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
-    {% if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and var('databricks_skip_optimize', 'false')|lower != 'true' %}
-      {% call statement('run_optimize_stmt') %}
+{%- macro databricks__optimize(relation) -%}
+  {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and 
+        var('databricks_skip_optimize', 'false')|lower != 'true' and 
+        config.get('file_format', 'delta') == 'delta' -%}
+    {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}
+      {%- call statement('run_optimize_stmt') -%}
         {{ get_optimize_sql(relation) }}
-      {% endcall %}
-    {% endif %}
-  {% endif %}
-{% endmacro %}
+      {%- endcall -%}
+    {%- endif -%}
+  {%- endif -%}
+{%- endmacro -%}
 
-{% macro get_optimize_sql(relation) %}
-  {% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
-     {%- set zorder = config.get('zorder', none) -%}
-    optimize {{ relation }}
-    {# TODO: predicates here? WHERE ...  #}
-    {% if zorder is sequence and zorder is not string %}
-      zorder by (
-        {%- for col in zorder -%}
+{%- macro get_optimize_sql(relation) %}
+  optimize {{ relation }}
+  {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
+    {%- if config.get('liquid_clustered_by', False) %}
+      {{ exceptions.warn("Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.") }}
+    {%- else %}
+      {%- set zorder = config.get('zorder', none) %}
+      {# TODO: predicates here? WHERE ...  #}
+      {%- if zorder is sequence and zorder is not string %}
+        zorder by (
+        {%- for col in zorder %}
         {{ col }}{% if not loop.last %}, {% endif %}
-        {%- endfor -%}
-      )
-    {% else %}
-      zorder by ({{zorder}})
-    {% endif %}
-  {% endif %}
-{% endmacro %}
+        {%- endfor %}
+        )
+      {%- else %}
+        zorder by ({{zorder}})
+      {%- endif %}
+    {%- endif %}
+  {%- endif %}
+{%- endmacro -%}
 
 {% macro databricks__list_relations_without_caching(schema_relation) %}
   {{ return(adapter.get_relations_without_caching(schema_relation)) }}

tests/integration/base.py

@@ -605,6 +605,12 @@ def _assertTablesEqualSql(self, relation_a, relation_b, columns=None):
 
         return sql
 
+    def assert_in_log(self, message: str) -> None:
+        log_file = os.path.join(self._logs_dir, "dbt.log")
+        with open(log_file, "r") as f:
+            log = f.read()
+            assert message.lower() in log.lower()
+
     def assertTablesEqual(
         self,
         table_a,

tests/integration/liquid_clustering/models/liquid_clustered.sql

@@ -0,0 +1,2 @@
+{{ config(materialized='incremental', liquid_clustered_by='id') }}
+select 1 as id, 'Joe' as name

tests/integration/liquid_clustering/test_liquid_clustering.py

@@ -0,0 +1,16 @@
+from tests.integration.base import DBTIntegrationTest, use_profile
+
+
+class TestLiquidClustering(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "liquid"
+
+    @property
+    def models(self):
+        return "models"
+
+    @use_profile("databricks_uc_sql_endpoint")
+    def test_liquid_clustering_databricks_uc_sql_endpoint(self):
+        self.run_dbt()
+        self.assert_in_log("optimize")

tests/integration/zorder/test_zorder.py

@@ -16,7 +16,8 @@ def project_config(self):
 
     def _test_zorder(self):
         self.run_dbt(["run"])
-        self.run_dbt(["run"])  # make sure it also run in incremental
+        self.run_dbt(["run"])
+        self.assert_in_log("zorder by")  # make sure it also run in incremental
 
     @use_profile("databricks_cluster")
     def test_zorder_databricks_cluster(self):

tests/unit/macros/test_adapters_macros.py

@@ -266,7 +266,7 @@ def test_macro_get_optimize_sql_multiple_args(self):
 
         self.assertEqual(
             sql,
-            ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo, bar)"),
+            ("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by ( foo, bar )"),
         )
 
     def test_macros_optimize_with_extraneous_info(self):