From 11c22ee08a99acfd0e7cb07d27c22700b06f60b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 12:10:53 -0600 Subject: [PATCH 1/5] fix: Append batch config if target supports the batch capability --- singer_sdk/target_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index d62bbbfd8..c31d692da 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -17,6 +17,7 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final from singer_sdk.helpers.capabilities import ( + BATCH_CONFIG, TARGET_SCHEMA_CONFIG, CapabilitiesEnum, PluginCapabilities, @@ -614,6 +615,9 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if TargetCapabilities.TARGET_SCHEMA in capabilities: _merge_missing(TARGET_SCHEMA_CONFIG, config_jsonschema) + if PluginCapabilities.BATCH in capabilities: + _merge_missing(BATCH_CONFIG, config_jsonschema) + super().append_builtin_config(config_jsonschema) pass From 619d68b9367a8d75ae3b04144907b4fcdf6364a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 12:51:04 -0600 Subject: [PATCH 2/5] Test about info --- singer_sdk/target_base.py | 35 +++++++++++++++++++++++++++++++--- tests/conftest.py | 8 ++++++++ tests/core/test_target_base.py | 19 ++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index c31d692da..b5a2ad5e2 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -571,6 +571,38 @@ def get_singer_command(cls: type[Target]) -> click.Command: return command + @classmethod + def append_builtin_config(cls: type[SQLTarget], config_jsonschema: dict) -> None: + """Appends built-in config to `config_jsonschema` if not already set. + + To customize or disable this behavior, developers may either override this class + method or override the `capabilities` property to disabled any unwanted + built-in capabilities. + + For all except very advanced use cases, we recommend leaving these + implementations "as-is", since this provides the most choice to users and is + the most "future proof" in terms of taking advantage of built-in capabilities + which may be added in the future. + + Args: + config_jsonschema: [description] + """ + + def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: + # Append any missing properties in the target with those from source. + for k, v in source_jsonschema["properties"].items(): + if k not in target_jsonschema["properties"]: + target_jsonschema["properties"][k] = v + + capabilities = cls.capabilities + + if PluginCapabilities.BATCH in capabilities: + _merge_missing(BATCH_CONFIG, config_jsonschema) + + super().append_builtin_config(config_jsonschema) + + pass + class SQLTarget(Target): """Target implementation for SQL destinations.""" @@ -615,9 +647,6 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if TargetCapabilities.TARGET_SCHEMA in capabilities: _merge_missing(TARGET_SCHEMA_CONFIG, config_jsonschema) - if PluginCapabilities.BATCH in capabilities: - _merge_missing(BATCH_CONFIG, config_jsonschema) - super().append_builtin_config(config_jsonschema) pass diff --git a/tests/conftest.py b/tests/conftest.py index 25319c015..023d9d7b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,12 +11,16 @@ from sqlalchemy import __version__ as sqlalchemy_version from singer_sdk import typing as th +from singer_sdk.helpers.capabilities import PluginCapabilities from singer_sdk.sinks import BatchSink from singer_sdk.target_base import Target if t.TYPE_CHECKING: from _pytest.config import Config + from singer_sdk.helpers.capabilities import CapabilitiesEnum + + SYSTEMS = {"linux", "darwin", "windows"} pytest_plugins = ("singer_sdk.testing.pytest_plugin",) @@ -103,6 +107,10 @@ class TargetMock(Target): name = "target-mock" config_jsonschema = th.PropertiesList().to_dict() default_sink_class = BatchSinkMock + capabilities: t.ClassVar[CapabilitiesEnum] = [ + *Target.capabilities, + PluginCapabilities.BATCH, + ] def __init__(self, *args, **kwargs): """Create the Mock target sync.""" diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index 1fd6b9a93..ed72b8496 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -5,6 +5,7 @@ import pytest from singer_sdk.exceptions import MissingKeyPropertiesError +from singer_sdk.helpers.capabilities import PluginCapabilities from tests.conftest import BatchSinkMock, TargetMock @@ -53,3 +54,21 @@ def test_validate_record(): # Test invalid record with pytest.raises(MissingKeyPropertiesError): sink._singer_validate_message({"name": "test"}) + + +def test_target_about_info(): + target = TargetMock() + about = target._get_about_info() + + assert about.capabilities == [ + PluginCapabilities.ABOUT, + PluginCapabilities.STREAM_MAPS, + PluginCapabilities.FLATTENING, + PluginCapabilities.BATCH, + ] + + assert "stream_maps" in about.settings["properties"] + assert "stream_map_config" in about.settings["properties"] + assert "flattening_enabled" in about.settings["properties"] + assert "flattening_max_depth" in about.settings["properties"] + assert "batch_config" in about.settings["properties"] From 0fc6b517ccc0da83dccc6e5acf5b7b22a2fd8e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 12:33:44 -0600 Subject: [PATCH 3/5] fix: Expose `add_record_metadata` as a builtin target setting --- singer_sdk/helpers/capabilities.py | 7 +++++++ singer_sdk/target_base.py | 3 +++ tests/core/test_target_base.py | 1 + 3 files changed, 11 insertions(+) diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index 55aff8ce3..f5b5fa305 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -99,6 +99,13 @@ description="The default target database schema name to use for all streams.", ), ).to_dict() +ADD_RECORD_METADATA_CONFIG = PropertiesList( + Property( + "add_record_metadata", + BooleanType(), + description="Add metadata to records.", + ), +).to_dict() class DeprecatedEnum(Enum): diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index b5a2ad5e2..d08fe42e0 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -17,6 +17,7 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final from singer_sdk.helpers.capabilities import ( + ADD_RECORD_METADATA_CONFIG, BATCH_CONFIG, TARGET_SCHEMA_CONFIG, CapabilitiesEnum, @@ -594,6 +595,8 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if k not in target_jsonschema["properties"]: target_jsonschema["properties"][k] = v + _merge_missing(ADD_RECORD_METADATA_CONFIG, config_jsonschema) + capabilities = cls.capabilities if PluginCapabilities.BATCH in capabilities: diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index ed72b8496..68e413d6a 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -72,3 +72,4 @@ def test_target_about_info(): assert "flattening_enabled" in about.settings["properties"] assert "flattening_max_depth" in about.settings["properties"] assert "batch_config" in about.settings["properties"] + assert "add_record_metadata" in about.settings["properties"] From a6a6ded3d4803761cf8ba292e250b3c731d75b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 12:58:03 -0600 Subject: [PATCH 4/5] Fix types --- singer_sdk/target_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index b5a2ad5e2..02e67d2c9 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -572,7 +572,7 @@ def get_singer_command(cls: type[Target]) -> click.Command: return command @classmethod - def append_builtin_config(cls: type[SQLTarget], config_jsonschema: dict) -> None: + def append_builtin_config(cls: type[Target], config_jsonschema: dict) -> None: """Appends built-in config to `config_jsonschema` if not already set. To customize or disable this behavior, developers may either override this class From f84c42ad1c252b4fec3862a2196d115401ad53fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 13:07:04 -0600 Subject: [PATCH 5/5] Fix more types --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 023d9d7b0..954c23fec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,7 +107,7 @@ class TargetMock(Target): name = "target-mock" config_jsonschema = th.PropertiesList().to_dict() default_sink_class = BatchSinkMock - capabilities: t.ClassVar[CapabilitiesEnum] = [ + capabilities: t.ClassVar[list[CapabilitiesEnum]] = [ *Target.capabilities, PluginCapabilities.BATCH, ]