From c12cab58327b2e45c735eeef734c9efd9e1a2fc1 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 19 Jun 2023 15:46:45 -0400 Subject: [PATCH 1/6] add tests for sql type parsing --- tests/core/test_typing.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 854669e5b..be0fb8340 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -17,6 +17,7 @@ PropertiesList, Property, StringType, + to_sql_type, ) if t.TYPE_CHECKING: @@ -292,3 +293,26 @@ def test_conform_primitives(): assert _conform_primitive_property(None, {"type": "boolean"}) is None assert _conform_primitive_property(0, {"type": "boolean"}) is False assert _conform_primitive_property(1, {"type": "boolean"}) is True + + + +import sqlalchemy +import pytest +@pytest.mark.parametrize( + "jsonschema_type,expected", + [ + ({'type': ['string', 'null']}, sqlalchemy.types.VARCHAR), + ({'type': ['integer', 'null']}, sqlalchemy.types.INTEGER), + ({'type': ['number', 'null']}, sqlalchemy.types.DECIMAL), + ({'type': ['boolean', 'null']}, sqlalchemy.types.BOOLEAN), + ({'type': "object", "properties": {}}, sqlalchemy.types.VARCHAR), + ({'type': "array"}, sqlalchemy.types.VARCHAR), + ({ "format": "date", "type": [ "string", "null" ] }, sqlalchemy.types.DATE), + ({ "format": "time", "type": [ "string", "null" ] }, sqlalchemy.types.TIME), + ({ "format": "date-time", "type": [ "string", "null" ] }, sqlalchemy.types.DATETIME), + ({"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, sqlalchemy.types.DATETIME), + ({ "anyOf": [ {"type": "integer"}, {"type": "null"}, ], }, sqlalchemy.types.INTEGER), + ] +) +def test_to_sql_type(jsonschema_type, expected): + assert isinstance(to_sql_type(jsonschema_type), expected) From 275c7359357be4273ebbf885a9a7e3747e5cd67c Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 19 Jun 2023 15:47:18 -0400 Subject: [PATCH 2/6] fix anyof sqltype parsing test --- singer_sdk/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index f2766944c..249f25a1d 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -825,7 +825,7 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo if jsonschema_type.get("type") in type_check: # noqa: PLR5501 return True - if any(t in type_check for t in jsonschema_type.get("anyOf", ())): + if any(_jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ())): return True return False From 9b13a09f7558d447fbdd09a592e9eb1b4596d6e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:49:08 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/core/test_typing.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index be0fb8340..24d09cb88 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -295,24 +295,31 @@ def test_conform_primitives(): assert _conform_primitive_property(1, {"type": "boolean"}) is True - -import sqlalchemy import pytest +import sqlalchemy + + @pytest.mark.parametrize( "jsonschema_type,expected", [ - ({'type': ['string', 'null']}, sqlalchemy.types.VARCHAR), - ({'type': ['integer', 'null']}, sqlalchemy.types.INTEGER), - ({'type': ['number', 'null']}, sqlalchemy.types.DECIMAL), - ({'type': ['boolean', 'null']}, sqlalchemy.types.BOOLEAN), - ({'type': "object", "properties": {}}, sqlalchemy.types.VARCHAR), - ({'type': "array"}, sqlalchemy.types.VARCHAR), - ({ "format": "date", "type": [ "string", "null" ] }, sqlalchemy.types.DATE), - ({ "format": "time", "type": [ "string", "null" ] }, sqlalchemy.types.TIME), - ({ "format": "date-time", "type": [ "string", "null" ] }, sqlalchemy.types.DATETIME), - ({"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, sqlalchemy.types.DATETIME), - ({ "anyOf": [ {"type": "integer"}, {"type": "null"}, ], }, sqlalchemy.types.INTEGER), - ] + ({"type": ["string", "null"]}, sqlalchemy.types.VARCHAR), + ({"type": ["integer", "null"]}, sqlalchemy.types.INTEGER), + ({"type": ["number", "null"]}, sqlalchemy.types.DECIMAL), + ({"type": ["boolean", "null"]}, sqlalchemy.types.BOOLEAN), + ({"type": "object", "properties": {}}, sqlalchemy.types.VARCHAR), + ({"type": "array"}, sqlalchemy.types.VARCHAR), + ({"format": "date", "type": ["string", "null"]}, sqlalchemy.types.DATE), + ({"format": "time", "type": ["string", "null"]}, sqlalchemy.types.TIME), + ( + {"format": "date-time", "type": ["string", "null"]}, + sqlalchemy.types.DATETIME, + ), + ( + {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, + sqlalchemy.types.DATETIME, + ), + ({"anyOf": [{"type": "integer"}, {"type": "null"}]}, sqlalchemy.types.INTEGER), + ], ) def test_to_sql_type(jsonschema_type, expected): assert isinstance(to_sql_type(jsonschema_type), expected) From 24fb96064b12f2c48b234dcf5aa06d3a70fed67d Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 19 Jun 2023 15:59:54 -0400 Subject: [PATCH 4/6] cleanup precommit errors --- singer_sdk/typing.py | 5 ++++- tests/core/test_typing.py | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 249f25a1d..42b316dcf 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -825,7 +825,10 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo if jsonschema_type.get("type") in type_check: # noqa: PLR5501 return True - if any(_jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ())): + if any( + _jsonschema_type_check(t, type_check) + for t in jsonschema_type.get("anyOf", ()) + ): return True return False diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 24d09cb88..0e90a91af 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -6,6 +6,8 @@ import logging import typing as t +import sqlalchemy + from singer_sdk.helpers._typing import ( TypeConformanceLevel, _conform_primitive_property, @@ -295,10 +297,6 @@ def test_conform_primitives(): assert _conform_primitive_property(1, {"type": "boolean"}) is True -import pytest -import sqlalchemy - - @pytest.mark.parametrize( "jsonschema_type,expected", [ From 585a68b098c00d2d717ee784ea4c0d12a396ebde Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:00:20 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- singer_sdk/typing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 42b316dcf..1e70f6201 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -826,8 +826,7 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo return True if any( - _jsonschema_type_check(t, type_check) - for t in jsonschema_type.get("anyOf", ()) + _jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ()) ): return True From a3372379b8d877fe4ec1d421f166e58949c90438 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 19 Jun 2023 16:02:32 -0400 Subject: [PATCH 6/6] update imports in tests --- tests/core/test_typing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 0e90a91af..b2cf9c691 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -4,8 +4,8 @@ import datetime import logging -import typing as t +import pytest import sqlalchemy from singer_sdk.helpers._typing import ( @@ -22,9 +22,6 @@ to_sql_type, ) -if t.TYPE_CHECKING: - import pytest - logger = logging.getLogger("log")