diff --git a/CHANGELOG.md b/CHANGELOG.md index 77eb72581..961fe01a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## dbt-spark 1.1.0 (TBD) +## dbt-spark 1.2.0 (April 28, 2022) + +### Fixes +- `adapter.get_columns_in_relation` (method) and `get_columns_in_relation` (macro) now return identical responses. The previous behavior of `get_columns_in_relation` (macro) is now represented by a new macro, `get_columns_in_relation_raw` ([#354](https://github.com/dbt-labs/dbt-spark/issues/354), [#355](https://github.com/dbt-labs/dbt-spark/pull/355)) + +## dbt-spark 1.1.0 (April 28, 2022) ### Features - Add session connection method ([#272](https://github.com/dbt-labs/dbt-spark/issues/272), [#279](https://github.com/dbt-labs/dbt-spark/pull/279)) diff --git a/dbt/adapters/spark/impl.py b/dbt/adapters/spark/impl.py index dd090a23b..4f7b9d4cc 100644 --- a/dbt/adapters/spark/impl.py +++ b/dbt/adapters/spark/impl.py @@ -23,7 +23,7 @@ logger = AdapterLogger("Spark") -GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" +GET_COLUMNS_IN_RELATION_RAW_MACRO_NAME = "spark__get_columns_in_relation_raw" LIST_SCHEMAS_MACRO_NAME = "list_schemas" LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" DROP_RELATION_MACRO_NAME = "drop_relation" @@ -225,7 +225,9 @@ def get_columns_in_relation(self, relation: Relation) -> List[SparkColumn]: # use get_columns_in_relation spark macro # which would execute 'describe extended tablename' query try: - rows: List[agate.Row] = super().get_columns_in_relation(relation) + rows: List[agate.Row] = self.execute_macro( + GET_COLUMNS_IN_RELATION_RAW_MACRO_NAME, kwargs={"relation": relation} + ) columns = self.parse_describe_extended(relation, rows) except dbt.exceptions.RuntimeException as e: # spark would throw error when table doesn't exist, where other diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index 22381d9ea..5322597ff 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -168,11 +168,15 @@ {%- endcall -%} {% endmacro %} -{% macro spark__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} +{% macro spark__get_columns_in_relation_raw(relation) -%} + {% call statement('get_columns_in_relation_raw', fetch_result=True) %} describe extended {{ relation.include(schema=(schema is not none)) }} {% endcall %} - {% do return(load_result('get_columns_in_relation').table) %} + {% do return(load_result('get_columns_in_relation_raw').table) %} +{% endmacro %} + +{% macro spark__get_columns_in_relation(relation) -%} + {{ return(adapter.get_columns_in_relation(relation)) }} {% endmacro %} {% macro spark__list_relations_without_caching(relation) %}