From 8f1dc035be88b3abf328ecc765d5958918c62934 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 11:40:11 -0700 Subject: [PATCH 1/8] Version bumps for 1.7.0 --- dbt/adapters/duckdb/__version__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/duckdb/__version__.py b/dbt/adapters/duckdb/__version__.py index a9851426..a55413d1 100644 --- a/dbt/adapters/duckdb/__version__.py +++ b/dbt/adapters/duckdb/__version__.py @@ -1 +1 @@ -version = "1.6.2" +version = "1.7.0" diff --git a/setup.py b/setup.py index 840c1e9b..3718858d 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.6.0", + "dbt-core==1.7.0rc1", "duckdb>=0.7.0", ], extras_require={ From 71d78255a3acb40fd9ce8a806f9f306fa517f00b Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 12:09:17 -0700 Subject: [PATCH 2/8] Placeholders for the not-yet-working date spine tests --- tests/functional/adapter/utils/test_utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index e25fcfb0..7e53d626 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -11,10 +11,14 @@ from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine from dbt.tests.adapter.utils.test_escape_single_quotes import ( BaseEscapeSingleQuotesQuote, ) from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect from dbt.tests.adapter.utils.test_last_day import BaseLastDay @@ -52,6 +56,9 @@ class TestDateDiff(BaseDateDiff): pass +class TestDateSpine(BaseDateSpine): + pass + class TestDateTrunc(BaseDateTrunc): pass @@ -60,6 +67,18 @@ class TestEscapeSingleQuotes(BaseEscapeSingleQuotesQuote): pass +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBetween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + class TestExcept(BaseExcept): pass From 8031fb41eae748f79bde67f4fa7e2c4d5780579f Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 12:30:53 -0700 Subject: [PATCH 3/8] Mark these tests as skipped for the moment --- tests/functional/adapter/utils/test_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 7e53d626..ffb0905e 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -56,6 +56,9 @@ class TestDateDiff(BaseDateDiff): pass +# Skipping this b/c the upstream utils test +# is irritatingly adapter-specific at the moment +@pytest.mark.skip class TestDateSpine(BaseDateSpine): pass @@ -71,6 +74,9 @@ class TestGenerateSeries(BaseGenerateSeries): pass +# Skipping this b/c the upstream utils test +# is irritatingly adapter-specific at the moment +@pytest.mark.skip class TestGetIntervalsBetween(BaseGetIntervalsBetween): pass From 971ef70cf82463237f61cf6decfa80fd8eb07895 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 12:17:36 -0700 Subject: [PATCH 4/8] Support delimiter options inside of the dbt-duckdb fast seed loader --- dbt/include/duckdb/macros/seed.sql | 3 ++- tests/functional/adapter/simple_seed/test_fast_seed.py | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbt/include/duckdb/macros/seed.sql b/dbt/include/duckdb/macros/seed.sql index 97f1bcfa..1fbc892c 100644 --- a/dbt/include/duckdb/macros/seed.sql +++ b/dbt/include/duckdb/macros/seed.sql @@ -10,8 +10,9 @@ {% macro duckdb__load_csv_rows(model, agate_table) %} {% if config.get('fast', true) %} {% set seed_file_path = adapter.get_seed_file_path(model) %} + {% set delimiter = config.get('delimiter', ',') %} {% set sql %} - COPY {{ this.render() }} FROM '{{ seed_file_path }}' (FORMAT CSV, HEADER TRUE) + COPY {{ this.render() }} FROM '{{ seed_file_path }}' (FORMAT CSV, HEADER TRUE, DELIMITER '{{ delimiter }}') {% endset %} {% do adapter.add_query(sql, abridge_sql_log=True) %} {{ return(sql) }} diff --git a/tests/functional/adapter/simple_seed/test_fast_seed.py b/tests/functional/adapter/simple_seed/test_fast_seed.py index ae3d7b4b..9ab9ed99 100644 --- a/tests/functional/adapter/simple_seed/test_fast_seed.py +++ b/tests/functional/adapter/simple_seed/test_fast_seed.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.adapter.simple_seed.test_seed import SeedTestBase +from dbt.tests.adapter.simple_seed.test_seed import SeedUniqueDelimiterTestBase from dbt.tests.util import ( run_dbt, ) @@ -14,4 +15,11 @@ def project_config_update(self): def test_simple_seed_fast(self, project): self._build_relations_for_test(project) - self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) \ No newline at end of file + self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) + + +class TestSeedWithUniqueDelimiter(SeedUniqueDelimiterTestBase): + def test_seed_with_unique_delimiter(self, project): + """Testing correct run of seeds with a unique delimiter (pipe in this case)""" + self._build_relations_for_test(project) + self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) From a79c482e619344e2c6982c95c815ce8439d96e03 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 12:20:13 -0700 Subject: [PATCH 5/8] Add in all of the seed delimiter tests --- .../adapter/simple_seed/test_fast_seed.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/functional/adapter/simple_seed/test_fast_seed.py b/tests/functional/adapter/simple_seed/test_fast_seed.py index 9ab9ed99..4c71d831 100644 --- a/tests/functional/adapter/simple_seed/test_fast_seed.py +++ b/tests/functional/adapter/simple_seed/test_fast_seed.py @@ -23,3 +23,29 @@ def test_seed_with_unique_delimiter(self, project): """Testing correct run of seeds with a unique delimiter (pipe in this case)""" self._build_relations_for_test(project) self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) + + +class TestSeedWithWrongDelimiter(SeedUniqueDelimiterTestBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": {"quote_columns": False, "delimiter": ";"}, + } + + def test_seed_with_wrong_delimiter(self, project): + """Testing failure of running dbt seed with a wrongly configured delimiter""" + seed_result = run_dbt(["seed"], expect_pass=False) + assert "syntax error" in seed_result.results[0].message.lower() + + +class TestSeedWithEmptyDelimiter(SeedUniqueDelimiterTestBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": {"quote_columns": False, "delimiter": ""}, + } + + def test_seed_with_empty_delimiter(self, project): + """Testing failure of running dbt seed with an empty configured delimiter value""" + seed_result = run_dbt(["seed"], expect_pass=False) + assert "compilation error" in seed_result.results[0].message.lower() From c59b08133364181467a3ca0bebb65ab9736b7796 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 13:07:04 -0700 Subject: [PATCH 6/8] Working test adapter version --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3bce0e89..8b8900bd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -dbt-tests-adapter==1.6.6 +dbt-tests-adapter==1.7.0rc1 boto3 mypy-boto3-glue From f338307c93e8cdd49f5a190cbffdb612e84d84a4 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 24 Oct 2023 13:36:27 -0700 Subject: [PATCH 7/8] Remove unused hologram-dependent unit tests --- tests/unit/utils.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 4dfa2833..82729023 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -7,7 +7,6 @@ from unittest import TestCase, mock from dbt.config.project import PartialProject -from hologram import ValidationError def normalize(path): @@ -121,33 +120,6 @@ def inject_adapter(value, plugin): FACTORY.adapters[key] = value -class ContractTestCase(TestCase): - ContractType = None - - def setUp(self): - self.maxDiff = None - super().setUp() - - def assert_to_dict(self, obj, dct): - self.assertEqual(obj.to_dict(), dct) - - def assert_from_dict(self, obj, dct, cls=None): - if cls is None: - cls = self.ContractType - self.assertEqual(cls.from_dict(dct), obj) - - def assert_symmetric(self, obj, dct, cls=None): - self.assert_to_dict(obj, dct) - self.assert_from_dict(obj, dct, cls) - - def assert_fails_validation(self, dct, cls=None): - if cls is None: - cls = self.ContractType - - with self.assertRaises(ValidationError): - cls.from_dict(dct) - - def generate_name_macros(package): from dbt.contracts.graph.parsed import ParsedMacro from dbt.node_types import NodeType From fae7aa7712983c5e9fc86067888783c6ace10ca8 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Fri, 3 Nov 2023 07:51:21 -0700 Subject: [PATCH 8/8] Update deps for the real dbt-core 1.7.0 stuff --- dev-requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8b8900bd..037fd9ae 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -dbt-tests-adapter==1.7.0rc1 +dbt-tests-adapter==1.7.0 boto3 mypy-boto3-glue diff --git a/setup.py b/setup.py index 3718858d..d24de3ee 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core==1.7.0rc1", + "dbt-core~=1.7.0", "duckdb>=0.7.0", ], extras_require={