From af0a5e51ee6838554c680a1a65aef72c6c4692ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Mon, 11 Sep 2023 08:56:18 -0600 Subject: [PATCH 1/3] fix: Retry SQLAlchemy engine creation for adapters without JSON SerDe support --- singer_sdk/connectors/sql.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index e05e359da..fb49b3587 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -319,12 +319,21 @@ def create_engine(self) -> Engine: Returns: A new SQLAlchemy Engine. """ - return sqlalchemy.create_engine( - self.sqlalchemy_url, - echo=False, - json_serializer=self.serialize_json, - json_deserializer=self.deserialize_json, - ) + try: + return sqlalchemy.create_engine( + self.sqlalchemy_url, + echo=False, + json_serializer=self.serialize_json, + json_deserializer=self.deserialize_json, + ) + except TypeError: + self.logger.exception( + "Retrying engine creation with fewer arguments due to TypeError.", + ) + return sqlalchemy.create_engine( + self.sqlalchemy_url, + echo=False, + ) def quote(self, name: str) -> str: """Quote a name if it needs quoting, using '.' as a name-part delimiter. From edaa6b1f1a022ed1132961bcc43c3af306015ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Mon, 11 Sep 2023 10:42:50 -0600 Subject: [PATCH 2/3] Add test --- samples/sample_custom_sql_adapter/__init__.py | 0 .../sample_custom_sql_adapter/connector.py | 19 +++++++++++++++++ tests/core/test_connector_sql.py | 21 ++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 samples/sample_custom_sql_adapter/__init__.py create mode 100644 samples/sample_custom_sql_adapter/connector.py diff --git a/samples/sample_custom_sql_adapter/__init__.py b/samples/sample_custom_sql_adapter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/sample_custom_sql_adapter/connector.py b/samples/sample_custom_sql_adapter/connector.py new file mode 100644 index 000000000..ac3a6e25c --- /dev/null +++ b/samples/sample_custom_sql_adapter/connector.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from sqlalchemy.engine.default import DefaultDialect + + +class CustomSQLDialect(DefaultDialect): + """Custom SQLite dialect that supports JSON.""" + + name = "myrdbms" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @classmethod + def import_dbapi(cls): + """Import the sqlite3 DBAPI.""" + import sqlite3 + + return sqlite3 diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 58ba59ec7..101d2d399 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -1,15 +1,19 @@ from __future__ import annotations +import typing as t from decimal import Decimal from unittest import mock import pytest import sqlalchemy -from sqlalchemy.dialects import sqlite +from sqlalchemy.dialects import registry, sqlite from singer_sdk.connectors import SQLConnector from singer_sdk.exceptions import ConfigValidationError +if t.TYPE_CHECKING: + from sqlalchemy.engine import Engine + def stringify(in_dict): return {k: str(v) for k, v in in_dict.items()} @@ -283,3 +287,18 @@ def test_engine_json_serialization(self, connector: SQLConnector): (1, {"x": Decimal("1.0")}), (2, {"x": Decimal("2.0"), "y": [1, 2, 3]}), ] + + +def test_adapter_without_json_serde(): + registry.register( + "myrdbms", + "samples.sample_custom_sql_adapter.connector", + "CustomSQLDialect", + ) + + class CustomConnector(SQLConnector): + def create_engine(self) -> Engine: + return super().create_engine() + + connector = CustomConnector(config={"sqlalchemy_url": "myrdbms:///"}) + connector.create_engine() From 1e3be4277748085992d7e3764e7174bc6ee8a187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Mon, 11 Sep 2023 10:46:43 -0600 Subject: [PATCH 3/3] Fix for SQLAlchemy 1 --- samples/sample_custom_sql_adapter/connector.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/samples/sample_custom_sql_adapter/connector.py b/samples/sample_custom_sql_adapter/connector.py index ac3a6e25c..6f7745a73 100644 --- a/samples/sample_custom_sql_adapter/connector.py +++ b/samples/sample_custom_sql_adapter/connector.py @@ -1,7 +1,12 @@ from __future__ import annotations +import typing as t + from sqlalchemy.engine.default import DefaultDialect +if t.TYPE_CHECKING: + from types import ModuleType + class CustomSQLDialect(DefaultDialect): """Custom SQLite dialect that supports JSON.""" @@ -17,3 +22,11 @@ def import_dbapi(cls): import sqlite3 return sqlite3 + + @classmethod + def dbapi(cls) -> ModuleType: # type: ignore[override] + """Return the DBAPI module. + + NOTE: This is a legacy method that will stop being used by SQLAlchemy at some point. + """ # noqa: E501 + return cls.import_dbapi()