From 713c84d309153885ba76ba62bfe5c5a48a59bcc4 Mon Sep 17 00:00:00 2001 From: Dan Norman Date: Tue, 8 Nov 2022 12:45:29 -0800 Subject: [PATCH 1/8] update schema_name porperty to ulitize default_target_schema --- singer_sdk/sinks/sql.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 6f8aed21e..9c29190d3 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -82,11 +82,31 @@ def schema_name(self) -> Optional[str]: Returns: The target schema name. """ + # Get the current SQL Dialect being used + target_sqla_dialect = self.connection.engine.dialect.name + # Look for a default_target_scheme in the configuraion fle + default_target_schema = self.config.get("default_target_schema", None) parts = self.stream_name.split("-") + + # 1) When default_target_scheme is in the configuration use it + # 2) if the streams are in - format use the + # stream + # 3) Return None if you don't find anything + if default_target_schema: + return default_target_schema + if len(parts) in {2, 3}: # Stream name is a two-part or three-part identifier. # Use the second-to-last part as the schema name. - return self.conform_name(parts[-2], "schema") + stream_schema = self.conform_name(parts[-2], "schema") + + # MS SQL Server has a public database role so the name is reserved + # and it can not be created as a schema. To avoid this common error + # we convert "public" to "dbo" if the target dialet is mssql + if target_sqla_dialect == "mssql" and stream_schema == "public": + return "dbo" + else: + return stream_schema # Schema name not detected. return None From fdec5eeb558e9abfba2cbd8410cd58260b51d2cb Mon Sep 17 00:00:00 2001 From: Dan Norman Date: Tue, 8 Nov 2022 15:32:45 -0800 Subject: [PATCH 2/8] activated TARGET_SCHEMA capability for SQLTarget Class and made it a bulit-in config opt --- singer_sdk/helpers/capabilities.py | 8 +++++ singer_sdk/target_base.py | 48 +++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index 82e33f7e6..33b6da7a1 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -12,6 +12,7 @@ ObjectType, PropertiesList, Property, + StringType, ) _EnumMemberT = TypeVar("_EnumMemberT") @@ -47,6 +48,13 @@ description="The max depth to flatten schemas.", ), ).to_dict() +TARGET_SCHEMA_CONFIG = PropertiesList( + Property( + "default_target_schema", + StringType(), + description="The Default schema to place all streams", + ), +).to_dict() class DeprecatedEnum(Enum): diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index a1ddd5e78..af795c5dc 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -17,7 +17,12 @@ from singer_sdk.helpers._batch import BaseBatchFileEncoding from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final -from singer_sdk.helpers.capabilities import CapabilitiesEnum, PluginCapabilities +from singer_sdk.helpers.capabilities import ( + TARGET_SCHEMA_CONFIG, + CapabilitiesEnum, + PluginCapabilities, + TargetCapabilities, +) from singer_sdk.io_base import SingerMessageType, SingerReader from singer_sdk.mapper import PluginMapper from singer_sdk.plugin_base import PluginBase @@ -569,4 +574,45 @@ def cli( class SQLTarget(Target): """Target implementation for SQL destinations.""" + @classproperty + def capabilities(self) -> List[CapabilitiesEnum]: + """Get target capabilities. + + Returns: + A list of capabilities supported by this target. + """ + sql_target_capabilities: List[CapabilitiesEnum] = super().capabilities + sql_target_capabilities.extend([TargetCapabilities.TARGET_SCHEMA]) + + return sql_target_capabilities + + def append_builtin_config(self, config_jsonschema: dict) -> None: + """Appends built-in config to `config_jsonschema` if not already set. + + To customize or disable this behavior, developers may either override this class + method or override the `capabilities` property to disabled any unwanted + built-in capabilities. + + For all except very advanced use cases, we recommend leaving these + implementations "as-is", since this provides the most choice to users and is + the most "future proof" in terms of taking advantage of built-in capabilities + which may be added in the future. + + Args: + config_jsonschema: [description] + """ + + def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: + # Append any missing properties in the target with those from source. + for k, v in source_jsonschema["properties"].items(): + if k not in target_jsonschema["properties"]: + target_jsonschema["properties"][k] = v + + capabilities = self.capabilities + + if TargetCapabilities.TARGET_SCHEMA in capabilities: + _merge_missing(TARGET_SCHEMA_CONFIG, config_jsonschema) + + super().append_builtin_config(config_jsonschema) + pass From d6ce9db10d108a6da22b6f41ad7c5053fc9495f7 Mon Sep 17 00:00:00 2001 From: Dan Norman Date: Tue, 8 Nov 2022 16:19:36 -0800 Subject: [PATCH 3/8] adding @classmethod back and flake8 tmp ignore --- singer_sdk/target_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index af795c5dc..ef7b830a3 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -586,7 +586,8 @@ def capabilities(self) -> List[CapabilitiesEnum]: return sql_target_capabilities - def append_builtin_config(self, config_jsonschema: dict) -> None: + @classmethod + def append_builtin_config(self, config_jsonschema: dict) -> None: # noqa: ANN102 """Appends built-in config to `config_jsonschema` if not already set. To customize or disable this behavior, developers may either override this class From d24bdad413492559c1ae8d97a2f4dd4ab24f0381 Mon Sep 17 00:00:00 2001 From: BuzzCutNorman Date: Wed, 9 Nov 2022 08:46:59 -0800 Subject: [PATCH 4/8] update default_target_schema description be more specific --- singer_sdk/helpers/capabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index 33b6da7a1..17bae7ef4 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -52,7 +52,7 @@ Property( "default_target_schema", StringType(), - description="The Default schema to place all streams", + description="The default target database schema name to use for all streams.", ), ).to_dict() From afb01c75eb5e085f439797d164ee2a9eb5e6584c Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 9 Nov 2022 13:46:51 -0600 Subject: [PATCH 5/8] `List` -> `list` --- singer_sdk/target_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index 66e3a16fc..f2bd0f7a0 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -577,13 +577,13 @@ class SQLTarget(Target): """Target implementation for SQL destinations.""" @classproperty - def capabilities(self) -> List[CapabilitiesEnum]: + def capabilities(self) -> list[CapabilitiesEnum]: """Get target capabilities. Returns: A list of capabilities supported by this target. """ - sql_target_capabilities: List[CapabilitiesEnum] = super().capabilities + sql_target_capabilities: list[CapabilitiesEnum] = super().capabilities sql_target_capabilities.extend([TargetCapabilities.TARGET_SCHEMA]) return sql_target_capabilities From 38d76130b89f03001773806d84be4221e6501532 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 9 Nov 2022 13:51:24 -0600 Subject: [PATCH 6/8] Annotate `default_target_schema` --- singer_sdk/sinks/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 9c29190d3..8206e287f 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -85,7 +85,7 @@ def schema_name(self) -> Optional[str]: # Get the current SQL Dialect being used target_sqla_dialect = self.connection.engine.dialect.name # Look for a default_target_scheme in the configuraion fle - default_target_schema = self.config.get("default_target_schema", None) + default_target_schema: str = self.config.get("default_target_schema", None) parts = self.stream_name.split("-") # 1) When default_target_scheme is in the configuration use it From 4f2c48fd1658127bf680faf5d9358508ebac281e Mon Sep 17 00:00:00 2001 From: BuzzCutNorman Date: Wed, 9 Nov 2022 14:24:09 -0800 Subject: [PATCH 7/8] removing mssql target schema error fix --- singer_sdk/sinks/sql.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 9c29190d3..7663d0919 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -82,8 +82,6 @@ def schema_name(self) -> Optional[str]: Returns: The target schema name. """ - # Get the current SQL Dialect being used - target_sqla_dialect = self.connection.engine.dialect.name # Look for a default_target_scheme in the configuraion fle default_target_schema = self.config.get("default_target_schema", None) parts = self.stream_name.split("-") @@ -99,14 +97,7 @@ def schema_name(self) -> Optional[str]: # Stream name is a two-part or three-part identifier. # Use the second-to-last part as the schema name. stream_schema = self.conform_name(parts[-2], "schema") - - # MS SQL Server has a public database role so the name is reserved - # and it can not be created as a schema. To avoid this common error - # we convert "public" to "dbo" if the target dialet is mssql - if target_sqla_dialect == "mssql" and stream_schema == "public": - return "dbo" - else: - return stream_schema + return stream_schema # Schema name not detected. return None From 601c5e22db6e540be22f0ee11ee03c55a5ff5666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 9 Nov 2022 17:40:47 -0600 Subject: [PATCH 8/8] Fix annotation for SQLTarget.append_builtin_config --- poetry.lock | 16 ---------------- singer_sdk/target_base.py | 4 ++-- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/poetry.lock b/poetry.lock index ea46533aa..d29d1fc7d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1370,23 +1370,14 @@ binaryornot = [ {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, ] black = [ - {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, - {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, - {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, - {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, - {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, - {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, - {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, - {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, - {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, @@ -2022,13 +2013,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index f2bd0f7a0..7462fb724 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -589,7 +589,7 @@ def capabilities(self) -> list[CapabilitiesEnum]: return sql_target_capabilities @classmethod - def append_builtin_config(self, config_jsonschema: dict) -> None: # noqa: ANN102 + def append_builtin_config(cls: type[SQLTarget], config_jsonschema: dict) -> None: """Appends built-in config to `config_jsonschema` if not already set. To customize or disable this behavior, developers may either override this class @@ -611,7 +611,7 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if k not in target_jsonschema["properties"]: target_jsonschema["properties"][k] = v - capabilities = self.capabilities + capabilities = cls.capabilities if TargetCapabilities.TARGET_SCHEMA in capabilities: _merge_missing(TARGET_SCHEMA_CONFIG, config_jsonschema)