From aaaf165b4353e9ad4778ddd4911841932af6de3c Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Mon, 18 Jul 2022 15:19:14 -0700 Subject: [PATCH 1/3] Add new basic tests TestDocsGenerateDatabricks and TestDocsGenReferencesDatabricks. --- dbt/adapters/databricks/impl.py | 4 +- tests/conftest.py | 6 +++ tests/functional/adapter/test_basic.py | 69 ++++++++++++++++++++++++-- tests/unit/test_adapter.py | 22 ++++---- 4 files changed, 85 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index fd456f2a..0645632f 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -154,7 +154,7 @@ def parse_describe_extended( table_owner=str(metadata.get(KEY_TABLE_OWNER)), table_stats=table_stats, column=column["col_name"], - column_index=idx, + column_index=(idx + 1), dtype=column["data_type"], ) for idx, column in enumerate(rows) @@ -177,7 +177,7 @@ def parse_columns_from_information( table_schema=relation.schema, table_name=relation.table, table_type=relation.type, - column_index=match_num, + column_index=(match_num + 1), table_owner=owner, column=column_name, dtype=column_type, diff --git a/tests/conftest.py b/tests/conftest.py index 61e2e3ab..cd142c61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,12 @@ def dbt_profile_target(request): return get_databricks_cluster_target(profile_type) +@pytest.fixture(scope="session") +def is_uc(request): + profile_type = request.config.getoption("--profile") + return "_uc_" in profile_type + + @pytest.fixture(autouse=True) def skip_by_profile_type(request): profile_type = request.config.getoption("--profile") diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 2c74325b..003e45a1 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -1,8 +1,15 @@ +import pytest + +from dbt.tests.adapter.basic.expected_catalog import ( + base_expected_catalog, + expected_references_catalog, + no_stats, +) +from dbt.tests.util import AnyFloat, AnyString + from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( - BaseSingularTestsEphemeral, -) +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral from dbt.tests.adapter.basic.test_empty import BaseEmpty from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral from dbt.tests.adapter.basic.test_incremental import BaseIncremental @@ -10,6 +17,7 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences class TestSimpleMaterializationsDatabricks(BaseSimpleMaterializations): @@ -50,3 +58,58 @@ class TestSnapshotTimestampDatabricks(BaseSnapshotTimestamp): class TestBaseAdapterMethodDatabricks(BaseAdapterMethod): pass + + +class TestDocsGenerateDatabricks(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=AnyString(), + id_type="long", + text_type="string", + time_type="timestamp", + view_type="view", + table_type="table", + model_stats=no_stats(), + ) + + +class TestDocsGenReferencesDatabricks(BaseDocsGenReferences): + @pytest.fixture(scope="class") + def model_stats(self, is_uc): + if is_uc: + return no_stats() + else: + return { + "bytes": { + "description": None, + "id": "bytes", + "include": True, + "label": "bytes", + "value": AnyFloat(), + }, + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": True, + "description": "Indicates whether there are statistics for this table", + "include": False, + }, + } + + @pytest.fixture(scope="class") + def expected_catalog(self, project, model_stats): + return expected_references_catalog( + project, + role=AnyString(), + id_type="long", + text_type="string", + time_type="timestamp", + bigint_type="long", + view_type="view", + table_type="table", + model_stats=model_stats, + seed_stats=no_stats(), + view_summary_stats=no_stats(), + ) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index eab40758..1ef67588 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -243,7 +243,7 @@ def test_parse_relation(self): "table_type": rel_type, "table_owner": "root", "column": "col1", - "column_index": 0, + "column_index": 1, "dtype": "decimal(22,0)", "numeric_scale": None, "numeric_precision": None, @@ -260,7 +260,7 @@ def test_parse_relation(self): "table_type": rel_type, "table_owner": "root", "column": "col2", - "column_index": 1, + "column_index": 2, "dtype": "string", "numeric_scale": None, "numeric_precision": None, @@ -277,7 +277,7 @@ def test_parse_relation(self): "table_type": rel_type, "table_owner": "root", "column": "dt", - "column_index": 2, + "column_index": 3, "dtype": "date", "numeric_scale": None, "numeric_precision": None, @@ -294,7 +294,7 @@ def test_parse_relation(self): "table_type": rel_type, "table_owner": "root", "column": "struct_col", - "column_index": 3, + "column_index": 4, "dtype": "struct", "numeric_scale": None, "numeric_precision": None, @@ -368,7 +368,7 @@ def test_parse_relation_with_statistics(self): "table_type": rel_type, "table_owner": "root", "column": "col1", - "column_index": 0, + "column_index": 1, "dtype": "decimal(22,0)", "numeric_scale": None, "numeric_precision": None, @@ -436,7 +436,7 @@ def test_parse_columns_from_information_with_table_type_and_delta_provider(self) "table_type": rel_type, "table_owner": "root", "column": "col1", - "column_index": 0, + "column_index": 1, "dtype": "decimal(22,0)", "numeric_scale": None, "numeric_precision": None, @@ -457,7 +457,7 @@ def test_parse_columns_from_information_with_table_type_and_delta_provider(self) "table_type": rel_type, "table_owner": "root", "column": "struct_col", - "column_index": 3, + "column_index": 4, "dtype": "struct", "numeric_scale": None, "numeric_precision": None, @@ -521,7 +521,7 @@ def test_parse_columns_from_information_with_view_type(self): "table_type": rel_type, "table_owner": "root", "column": "col2", - "column_index": 1, + "column_index": 2, "dtype": "string", "numeric_scale": None, "numeric_precision": None, @@ -538,7 +538,7 @@ def test_parse_columns_from_information_with_view_type(self): "table_type": rel_type, "table_owner": "root", "column": "struct_col", - "column_index": 3, + "column_index": 4, "dtype": "struct", "numeric_scale": None, "numeric_precision": None, @@ -587,7 +587,7 @@ def test_parse_columns_from_information_with_table_type_and_parquet_provider(sel "table_type": rel_type, "table_owner": "root", "column": "dt", - "column_index": 2, + "column_index": 3, "dtype": "date", "numeric_scale": None, "numeric_precision": None, @@ -612,7 +612,7 @@ def test_parse_columns_from_information_with_table_type_and_parquet_provider(sel "table_type": rel_type, "table_owner": "root", "column": "struct_col", - "column_index": 3, + "column_index": 4, "dtype": "struct", "numeric_scale": None, "numeric_precision": None, From cf5dd34a5cf44170ff9749fd69802a7990e45e5f Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Wed, 20 Jul 2022 15:57:13 -0700 Subject: [PATCH 2/3] Fix. --- tests/conftest.py | 6 --- tests/functional/adapter/test_basic.py | 53 ++++++++++++-------------- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cd142c61..61e2e3ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,12 +25,6 @@ def dbt_profile_target(request): return get_databricks_cluster_target(profile_type) -@pytest.fixture(scope="session") -def is_uc(request): - profile_type = request.config.getoption("--profile") - return "_uc_" in profile_type - - @pytest.fixture(autouse=True) def skip_by_profile_type(request): profile_type = request.config.getoption("--profile") diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 003e45a1..96849d3a 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -3,9 +3,8 @@ from dbt.tests.adapter.basic.expected_catalog import ( base_expected_catalog, expected_references_catalog, - no_stats, ) -from dbt.tests.util import AnyFloat, AnyString +from dbt.tests.util import AnyInteger, AnyString from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests @@ -71,35 +70,13 @@ def expected_catalog(self, project): time_type="timestamp", view_type="view", table_type="table", - model_stats=no_stats(), + model_stats=_StatsLikeDict(), ) class TestDocsGenReferencesDatabricks(BaseDocsGenReferences): @pytest.fixture(scope="class") - def model_stats(self, is_uc): - if is_uc: - return no_stats() - else: - return { - "bytes": { - "description": None, - "id": "bytes", - "include": True, - "label": "bytes", - "value": AnyFloat(), - }, - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": True, - "description": "Indicates whether there are statistics for this table", - "include": False, - }, - } - - @pytest.fixture(scope="class") - def expected_catalog(self, project, model_stats): + def expected_catalog(self, project): return expected_references_catalog( project, role=AnyString(), @@ -109,7 +86,25 @@ def expected_catalog(self, project, model_stats): bigint_type="long", view_type="view", table_type="table", - model_stats=model_stats, - seed_stats=no_stats(), - view_summary_stats=no_stats(), + model_stats=_StatsLikeDict(), + ) + + +class _StatsLikeDict: + """Any stats-like dict. Use this in assert calls""" + + def __eq__(self, other): + return ( + isinstance(other, dict) + and "has_stats" in other + and ( + other["has_stats"] + == { + "id": "has_stats", + "label": "Has Stats?", + "value": AnyInteger(), + "description": "Indicates whether there are statistics for this table", + "include": False, + } + ) ) From 3a4f60734061940ef1751b50666b2cec2e3e89af Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Wed, 20 Jul 2022 16:03:32 -0700 Subject: [PATCH 3/3] Fix. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8191baf6..e93fb7e6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = linter, unit [testenv:linter] basepython = python3 commands = /bin/bash -c '$(which black) --config black.ini --check .' - /bin/bash -c '$(which flake8) --select=E,W,F --ignore=E203 --max-line-length=100 .' + /bin/bash -c '$(which flake8) --select=E,W,F --ignore=E203,W503 --max-line-length=100 .' /bin/bash -c '$(which mypy) --config-file mypy.ini --namespace-packages --explicit-package-bases dbt tests' passenv = DBT_* PYTEST_ADDOPTS deps =