From e357c9cc48de67dde71141c20d70a66b148b9076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 8 Aug 2023 17:47:20 -0600 Subject: [PATCH 1/4] fix: Pin `singer-sdk` to a working minor version --- poetry.lock | 20 +++++++++++--------- pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index 01a9417..61a9e4c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -653,13 +653,13 @@ asn1crypto = ">=1.5.1" [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] @@ -1209,19 +1209,19 @@ files = [ [[package]] name = "singer-sdk" -version = "0.23.0" +version = "0.30.0" description = "A framework for building Singer taps" optional = false python-versions = ">=3.7.1,<3.12" files = [ - {file = "singer_sdk-0.23.0-py3-none-any.whl", hash = "sha256:bb032ad9cc3805c9f840e5ca06c6208e5dea472b23ef6d9a324513fb51fc57f9"}, - {file = "singer_sdk-0.23.0.tar.gz", hash = "sha256:4bb62f23669568ea2f57a288d6c53e109bf945fb9e06c2ece4cb8331586415e3"}, + {file = "singer_sdk-0.30.0-py3-none-any.whl", hash = "sha256:ca8e002ab6fce56c4e6f3312655c7b68896d1d07f88a9efeb373e74c58ceb9b8"}, + {file = "singer_sdk-0.30.0.tar.gz", hash = "sha256:73300d1c1bcb048b49a7f28c71ae26c9d309fdf3375e65940747b05c8a6551b5"}, ] [package.dependencies] backoff = ">=2.0.0,<3.0" click = ">=8.0,<9.0" -cryptography = ">=3.4.6,<41.0.0" +cryptography = ">=3.4.6,<42.0.0" fs = ">=2.4.16,<3.0.0" importlib-metadata = {version = "<5.0.0", markers = "python_version < \"3.8\""} importlib-resources = {version = "5.12.0", markers = "python_version < \"3.9\""} @@ -1230,6 +1230,7 @@ joblib = ">=1.0.1,<2.0.0" jsonpath-ng = ">=1.5.3,<2.0.0" jsonschema = ">=4.16.0,<5.0.0" memoization = ">=0.3.2,<0.5.0" +packaging = ">=23.1" pendulum = ">=2.1.0,<3.0.0" PyJWT = ">=2.4,<3.0" pytest = {version = ">=7.2.1,<8.0.0", optional = true, markers = "extra == \"testing\""} @@ -1241,9 +1242,10 @@ requests = ">=2.25.1,<3.0.0" simplejson = ">=3.17.6,<4.0.0" sqlalchemy = ">=1.4,<2.0" typing-extensions = ">=4.2.0,<5.0.0" +urllib3 = ">=1.26,<2" [package.extras] -docs = ["furo (>=2022.12.7,<2024.0.0)", "myst-parser (>=0.17.2,<1.1.0)", "sphinx (>=4.5,<6.0)", "sphinx-autobuild (>=2021.3.14,<2022.0.0)", "sphinx-copybutton (>=0.3.1,<0.6.0)", "sphinx-reredirects (>=0.1.1,<0.2.0)"] +docs = ["furo (>=2022.12.7,<2024.0.0)", "myst-parser (>=0.17.2,<1.1.0)", "sphinx (>=4.5,<6.0)", "sphinx-autobuild (>=2021.3.14,<2022.0.0)", "sphinx-copybutton (>=0.3.1,<0.6.0)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-reredirects (>=0.1.1,<0.2.0)"] s3 = ["fs-s3fs (>=1.1.1,<2.0.0)"] testing = ["pytest (>=7.2.1,<8.0.0)", "pytest-durations (>=1.2.0,<2.0.0)"] @@ -1580,4 +1582,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7.1,<3.12" -content-hash = "706d29e63085dcadf3f1cbd02bebda94332eb9cd7b5b0bffefdadd9884aad6e8" +content-hash = "b42530c0b8818979de69c5c5a92e9d1a2dba02510721385207302fd4713425c5" diff --git a/pyproject.toml b/pyproject.toml index bd5a562..8da2e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.7.1,<3.12" requests = "^2.25.1" -singer-sdk = ">=0.23.0" +singer-sdk = "~=0.30.0" snowflake-sqlalchemy = "^1.4.7" snowflake-connector-python = "^3.0.2" sqlalchemy = "<2" From 814e09d1f038542ca918ca705817a612c677897a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 8 Aug 2023 17:51:14 -0600 Subject: [PATCH 2/4] feat: Bump `singer-sdk` to v0.31.0 --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- tap_snowflake/client.py | 19 +++++++++---------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index 61a9e4c..b22ce53 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1209,13 +1209,13 @@ files = [ [[package]] name = "singer-sdk" -version = "0.30.0" +version = "0.31.0" description = "A framework for building Singer taps" optional = false python-versions = ">=3.7.1,<3.12" files = [ - {file = "singer_sdk-0.30.0-py3-none-any.whl", hash = "sha256:ca8e002ab6fce56c4e6f3312655c7b68896d1d07f88a9efeb373e74c58ceb9b8"}, - {file = "singer_sdk-0.30.0.tar.gz", hash = "sha256:73300d1c1bcb048b49a7f28c71ae26c9d309fdf3375e65940747b05c8a6551b5"}, + {file = "singer_sdk-0.31.0-py3-none-any.whl", hash = "sha256:aa9329bd7e26f9e2bd59332468e064dd38f6df2dd6b7445e047d8008acd0dd65"}, + {file = "singer_sdk-0.31.0.tar.gz", hash = "sha256:de8fdf00967dc13b4073dddad499799df2a1273806406b765ba55b671ea18b8a"}, ] [package.dependencies] @@ -1240,7 +1240,7 @@ pytz = ">=2022.2.1,<2024.0.0" PyYAML = ">=6.0,<7.0" requests = ">=2.25.1,<3.0.0" simplejson = ">=3.17.6,<4.0.0" -sqlalchemy = ">=1.4,<2.0" +sqlalchemy = ">=1.4,<3.0" typing-extensions = ">=4.2.0,<5.0.0" urllib3 = ">=1.26,<2" @@ -1582,4 +1582,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7.1,<3.12" -content-hash = "b42530c0b8818979de69c5c5a92e9d1a2dba02510721385207302fd4713425c5" +content-hash = "7b1e6fca059b8b927b932ebf8caee33f23e8cb084e92e88e5b0587fbd8a2853f" diff --git a/pyproject.toml b/pyproject.toml index 8da2e5f..1e4b394 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.7.1,<3.12" requests = "^2.25.1" -singer-sdk = "~=0.30.0" +singer-sdk = "~=0.31.0" snowflake-sqlalchemy = "^1.4.7" snowflake-connector-python = "^3.0.2" sqlalchemy = "<2" diff --git a/tap_snowflake/client.py b/tap_snowflake/client.py index 0e4c375..f00b999 100644 --- a/tap_snowflake/client.py +++ b/tap_snowflake/client.py @@ -17,6 +17,7 @@ from singer_sdk.helpers._batch import BaseBatchFileEncoding, BatchConfig from singer_sdk.streams.core import REPLICATION_FULL_TABLE, REPLICATION_INCREMENTAL from snowflake.sqlalchemy import URL +from sqlalchemy.engine import Engine from sqlalchemy.sql import text @@ -68,17 +69,15 @@ def get_sqlalchemy_url(cls, config: dict) -> str: return URL(**params) - def create_sqlalchemy_engine(self) -> sqlalchemy.engine.Engine: - """Return a new SQLAlchemy engine using the provided config. - - Developers can generally override just one of the following: - `sqlalchemy_engine`, sqlalchemy_url`. - - Returns: - A newly created SQLAlchemy engine object. - """ + def create_engine(self) -> Engine: return sqlalchemy.create_engine( - self.sqlalchemy_url, echo=False, pool_timeout=10 + self.sqlalchemy_url, + echo=False, + pool_timeout=10, + # TODO: Enable these when `snowflake-sqlalchemy` supports it. + # https://github.com/snowflakedb/snowflake-sqlalchemy/issues/433 + # json_serializer=self.serialize_json, + # json_deserializer=self.deserialize_json, ) # overridden to filter out the information_schema from catalog discovery From ea5de298968549872758ecd477ac47ecc4a16518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Wed, 9 Aug 2023 11:20:20 -0600 Subject: [PATCH 3/4] Update catalog --- tests/catalog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/catalog.json b/tests/catalog.json index 62e5e69..c3f4c5f 100644 --- a/tests/catalog.json +++ b/tests/catalog.json @@ -378,7 +378,7 @@ "type": ["number"] }, "o_orderdate": { - "format": "date", + "format": "date-time", "type": ["string"] }, "o_orderpriority": { From e426815a5d915e8e2c5e6f0a40db9d027b369e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Wed, 9 Aug 2023 11:20:20 -0600 Subject: [PATCH 4/4] Update catalog --- tests/catalog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/catalog.json b/tests/catalog.json index 62e5e69..c3f4c5f 100644 --- a/tests/catalog.json +++ b/tests/catalog.json @@ -378,7 +378,7 @@ "type": ["number"] }, "o_orderdate": { - "format": "date", + "format": "date-time", "type": ["string"] }, "o_orderpriority": {