From 55f387523505c74d70c216ab9769ef3c5120bb90 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 6 Nov 2023 10:38:16 -0500 Subject: [PATCH 1/3] Seed test, modifications to get new agate Integer type working --- .../unreleased/Fixes-20231031-144837.yaml | 6 +++++ core/dbt/adapters/base/impl.py | 14 +++++++++++ core/dbt/adapters/sql/impl.py | 4 ++++ tests/functional/seeds/test_seeds.py | 23 +++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 .changes/unreleased/Fixes-20231031-144837.yaml create mode 100644 tests/functional/seeds/test_seeds.py diff --git a/.changes/unreleased/Fixes-20231031-144837.yaml b/.changes/unreleased/Fixes-20231031-144837.yaml new file mode 100644 index 00000000000..177c3ed2db7 --- /dev/null +++ b/.changes/unreleased/Fixes-20231031-144837.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix exception running empty seed file +time: 2023-10-31T14:48:37.774871-04:00 +custom: + Author: gshank + Issue: "8895" diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 62ac303ca14..739d85e24b3 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -51,6 +51,7 @@ get_column_value_uncased, merge_tables, table_from_rows, + Integer, ) from dbt.clients.jinja import MacroGenerator from dbt.contracts.graph.manifest import Manifest, MacroManifest @@ -962,6 +963,18 @@ def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: """ raise NotImplementedError("`convert_number_type` is not implemented for this adapter!") + @classmethod + @abc.abstractmethod + def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: + """Return the type in the database that best maps to the agate.Number + type for the given agate table and column index. + + :param agate_table: The table + :param col_idx: The index into the agate table for the column. + :return: The name of the type in the database + """ + raise NotImplementedError("`convert_integer_type` is not implemented for this adapter!") + @classmethod @abc.abstractmethod def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: @@ -1019,6 +1032,7 @@ def convert_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: def convert_agate_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: agate_type: Type = agate_table.column_types[col_idx] conversions: List[Tuple[Type, Callable[..., str]]] = [ + (Integer, cls.convert_integer_type), (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index b74eb02d991..de4c109bb54 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -75,6 +75,10 @@ def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] return "float8" if decimals else "integer" + @classmethod + def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "integer" + @classmethod def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "boolean" diff --git a/tests/functional/seeds/test_seeds.py b/tests/functional/seeds/test_seeds.py new file mode 100644 index 00000000000..aecc22f2bf3 --- /dev/null +++ b/tests/functional/seeds/test_seeds.py @@ -0,0 +1,23 @@ +import pytest +from dbt.tests.util import run_dbt + +# This is for seed tests on dbt-core code, not adapter code + + +class TestEmptySeed: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"empty_with_header.csv": "a,b,c"} + + def test_empty_seeds(self, project): + # Should create an empty table and not fail + results = run_dbt(["seed"]) + assert len(results) == 1 From d72790ed9fdf99252839abad06366c174d485dfb Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 6 Nov 2023 18:02:56 -0500 Subject: [PATCH 2/3] Give convert_integer_type a default, adjust mock_adapter --- core/dbt/adapters/base/impl.py | 3 +-- tests/unit/mock_adapter.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 739d85e24b3..60c2dd397b2 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -964,7 +964,6 @@ def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: raise NotImplementedError("`convert_number_type` is not implemented for this adapter!") @classmethod - @abc.abstractmethod def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.Number type for the given agate table and column index. @@ -973,7 +972,7 @@ def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise NotImplementedError("`convert_integer_type` is not implemented for this adapter!") + return "integer" @classmethod @abc.abstractmethod diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index d3bdf87b2e4..8858542619b 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -55,6 +55,9 @@ def convert_text_type(self, *args, **kwargs): def convert_number_type(self, *args, **kwargs): return self.responder.convert_number_type(*args, **kwargs) + def convert_integer_type(self, *args, **kwargs): + return self.responder.convert_integer_type(*args, **kwargs) + def convert_boolean_type(self, *args, **kwargs): return self.responder.convert_boolean_type(*args, **kwargs) From bce7d56eb2c1536a516af811b56756083486c99c Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 7 Nov 2023 09:19:52 -0500 Subject: [PATCH 3/3] Expand changelog line and move test to adapter zone --- .../unreleased/Fixes-20231031-144837.yaml | 2 +- .../tests/adapter/simple_seed/test_seed.py | 23 +++++++++++++++++++ tests/functional/seeds/test_seeds.py | 23 ------------------- 3 files changed, 24 insertions(+), 24 deletions(-) delete mode 100644 tests/functional/seeds/test_seeds.py diff --git a/.changes/unreleased/Fixes-20231031-144837.yaml b/.changes/unreleased/Fixes-20231031-144837.yaml index 177c3ed2db7..64b15e29dc9 100644 --- a/.changes/unreleased/Fixes-20231031-144837.yaml +++ b/.changes/unreleased/Fixes-20231031-144837.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: Fix exception running empty seed file +body: Fix compilation exception running empty seed file and support new Integer agate data_type time: 2023-10-31T14:48:37.774871-04:00 custom: Author: gshank diff --git a/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py b/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py index 74f2f07a4dc..3e42cd4c0d0 100644 --- a/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py +++ b/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py @@ -365,3 +365,26 @@ def seeds(self, test_data_dir): def test_simple_seed(self, project): results = run_dbt(["seed"]) assert len(results) == 3 + + +class BaseTestEmptySeed: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"empty_with_header.csv": "a,b,c"} + + def test_empty_seeds(self, project): + # Should create an empty table and not fail + results = run_dbt(["seed"]) + assert len(results) == 1 + + +class TestEmptySeed(BaseTestEmptySeed): + pass diff --git a/tests/functional/seeds/test_seeds.py b/tests/functional/seeds/test_seeds.py deleted file mode 100644 index aecc22f2bf3..00000000000 --- a/tests/functional/seeds/test_seeds.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest -from dbt.tests.util import run_dbt - -# This is for seed tests on dbt-core code, not adapter code - - -class TestEmptySeed: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "quote_columns": False, - }, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"empty_with_header.csv": "a,b,c"} - - def test_empty_seeds(self, project): - # Should create an empty table and not fail - results = run_dbt(["seed"]) - assert len(results) == 1