diff --git a/poetry.lock b/poetry.lock index 20701b6..689206e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = "*" files = [ @@ -16,7 +15,6 @@ files = [ name = "asn1crypto" version = "1.5.1" description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -category = "main" optional = false python-versions = "*" files = [ @@ -28,7 +26,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -50,7 +47,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -62,7 +58,6 @@ files = [ name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -74,7 +69,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -151,7 +145,6 @@ pycparser = "*" name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -234,14 +227,13 @@ files = [ [[package]] name = "click" -version = "8.1.3" +version = "8.1.4" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.4-py3-none-any.whl", hash = "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3"}, + {file = "click-8.1.4.tar.gz", hash = "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37"}, ] [package.dependencies] @@ -252,7 +244,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -264,7 +255,6 @@ files = [ name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -337,7 +327,6 @@ toml = ["tomli"] name = "cryptography" version = "40.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -379,7 +368,6 @@ tox = ["tox"] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -391,7 +379,6 @@ files = [ name = "exceptiongroup" version = "1.1.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -406,7 +393,6 @@ test = ["pytest (>=6)"] name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -422,7 +408,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "fs" version = "2.4.16" description = "Python's filesystem abstraction layer" -category = "main" optional = false python-versions = "*" files = [ @@ -442,7 +427,6 @@ scandir = ["scandir (>=1.5,<2.0)"] name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -516,7 +500,6 @@ test = ["objgraph", "psutil"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -528,7 +511,6 @@ files = [ name = "importlib-metadata" version = "4.13.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -549,7 +531,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -568,7 +549,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "inflection" version = "0.5.1" description = "A port of Ruby on Rails inflector to Python" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -580,7 +560,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -592,7 +571,6 @@ files = [ name = "joblib" version = "1.3.1" description = "Lightweight pipelining with Python functions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -604,7 +582,6 @@ files = [ name = "jsonpath-ng" version = "1.5.3" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -category = "main" optional = false python-versions = "*" files = [ @@ -622,7 +599,6 @@ six = "*" name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -646,7 +622,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "memoization" version = "0.4.0" description = "A powerful caching library for Python, with TTL support and multiple algorithm options. (https://github.com/lonelyenvoy/python-memoization)" -category = "main" optional = false python-versions = ">=3, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" files = [ @@ -657,7 +632,6 @@ files = [ name = "oscrypto" version = "1.3.0" description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." -category = "main" optional = false python-versions = "*" files = [ @@ -672,7 +646,6 @@ asn1crypto = ">=1.5.1" name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -684,7 +657,6 @@ files = [ name = "pendulum" version = "2.1.2" description = "Python datetimes made easy" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -719,7 +691,6 @@ pytzdata = ">=2020.1" name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -731,7 +702,6 @@ files = [ name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -750,7 +720,6 @@ testing = ["pytest", "pytest-benchmark"] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "main" optional = false python-versions = "*" files = [ @@ -762,7 +731,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -774,7 +742,6 @@ files = [ name = "pycryptodomex" version = "3.18.0" description = "Cryptographic library for Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -816,7 +783,6 @@ files = [ name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -837,7 +803,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pyopenssl" version = "23.2.0" description = "Python wrapper module around the OpenSSL library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -856,7 +821,6 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -893,7 +857,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -917,7 +880,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-durations" version = "1.2.0" description = "Pytest plugin reporting fixtures and test functions execution time." -category = "dev" optional = false python-versions = ">=3.6.2" files = [ @@ -932,7 +894,6 @@ pytest = ">=4.6" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -947,7 +908,6 @@ six = ">=1.5" name = "python-dotenv" version = "0.21.1" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -962,7 +922,6 @@ cli = ["click (>=5.0)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -974,7 +933,6 @@ files = [ name = "pytzdata" version = "2020.1" description = "The Olson timezone database for Python." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -986,7 +944,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1036,7 +993,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1058,7 +1014,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1075,7 +1030,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "simplejson" version = "3.19.1" description = "Simple, fast, extensible JSON encoder/decoder for Python" -category = "main" optional = false python-versions = ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1168,14 +1122,13 @@ files = [ [[package]] name = "singer-sdk" -version = "0.28.0" +version = "0.29.0" description = "A framework for building Singer taps" -category = "main" optional = false python-versions = ">=3.7.1,<3.12" files = [ - {file = "singer_sdk-0.28.0-py3-none-any.whl", hash = "sha256:04463f1f429cf1913de5890068b4cac300b25d2b84ca03a974f2414e7dd35ff4"}, - {file = "singer_sdk-0.28.0.tar.gz", hash = "sha256:3cf50f91f0cb8e8bd1db3baf025ffe889a3f70d83a94f944b2a941ca5c7a7055"}, + {file = "singer_sdk-0.29.0-py3-none-any.whl", hash = "sha256:b24dabd0cfd820e37efaedfc0bc0a131b75ec486131e2c87aa20794dde858171"}, + {file = "singer_sdk-0.29.0.tar.gz", hash = "sha256:66ec185831d8049fac4c2c686b56dd91da9a17becf6194afdada0b0d6331ce37"}, ] [package.dependencies] @@ -1204,7 +1157,7 @@ typing-extensions = ">=4.2.0,<5.0.0" urllib3 = ">=1.26,<2" [package.extras] -docs = ["furo (>=2022.12.7,<2024.0.0)", "myst-parser (>=0.17.2,<1.1.0)", "sphinx (>=4.5,<6.0)", "sphinx-autobuild (>=2021.3.14,<2022.0.0)", "sphinx-copybutton (>=0.3.1,<0.6.0)", "sphinx-reredirects (>=0.1.1,<0.2.0)"] +docs = ["furo (>=2022.12.7,<2024.0.0)", "myst-parser (>=0.17.2,<1.1.0)", "sphinx (>=4.5,<6.0)", "sphinx-autobuild (>=2021.3.14,<2022.0.0)", "sphinx-copybutton (>=0.3.1,<0.6.0)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-reredirects (>=0.1.1,<0.2.0)"] s3 = ["fs-s3fs (>=1.1.1,<2.0.0)"] testing = ["pytest (>=7.2.1,<8.0.0)", "pytest-durations (>=1.2.0,<2.0.0)"] @@ -1212,7 +1165,6 @@ testing = ["pytest (>=7.2.1,<8.0.0)", "pytest-durations (>=1.2.0,<2.0.0)"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1224,7 +1176,6 @@ files = [ name = "snowflake-connector-python" version = "3.0.4" description = "Snowflake Connector for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1284,7 +1235,6 @@ secure-local-storage = ["keyring (!=16.1.0,<24.0.0)"] name = "snowflake-sqlalchemy" version = "1.4.7" description = "Snowflake SQLAlchemy Dialect" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1305,7 +1255,6 @@ pandas = ["snowflake-connector-python[pandas] (<4.0.0)"] name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -category = "main" optional = false python-versions = "*" files = [ @@ -1317,10 +1266,11 @@ files = [ name = "sqlalchemy" version = "1.4.49" description = "Database Abstraction Library" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ + {file = "SQLAlchemy-1.4.49-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e126cf98b7fd38f1e33c64484406b78e937b1a280e078ef558b95bf5b6895f6"}, + {file = "SQLAlchemy-1.4.49-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:03db81b89fe7ef3857b4a00b63dedd632d6183d4ea5a31c5d8a92e000a41fc71"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:95b9df9afd680b7a3b13b38adf6e3a38995da5e162cc7524ef08e3be4e5ed3e1"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63e43bf3f668c11bb0444ce6e809c1227b8f067ca1068898f3008a273f52b09"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f835c050ebaa4e48b18403bed2c0fda986525896efd76c245bdd4db995e51a4c"}, @@ -1360,7 +1310,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] @@ -1388,7 +1338,6 @@ sqlcipher = ["sqlcipher3-binary"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1400,7 +1349,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1412,7 +1360,6 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1429,7 +1376,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1444,4 +1390,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7.1" -content-hash = "7114ca163fa30122722e006115bca3e7e06badddb5d7c7065446104668476fc5" +content-hash = "e06d0824d560413615b607317d341c0bbadb3fed49d03a4fc83be0e04e537e76" diff --git a/pyproject.toml b/pyproject.toml index 693c4b6..0eeb5f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,13 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" requests = "^2.31.0" -snowflake-sqlalchemy = "^1.4.1" -singer-sdk = "0.28.0" +snowflake-sqlalchemy = "^1.4.7" +singer-sdk = "0.29.0" +cryptography = "^40.0.2" [tool.poetry.dev-dependencies] pytest = "^7.3.1" -singer-sdk = { version="0.28.0", extras = ["testing"] } +singer-sdk = { version="0.29.0", extras = ["testing"] } [tool.poetry.group.dev.dependencies] coverage = "^7.2.7" diff --git a/target_snowflake/connector.py b/target_snowflake/connector.py index e97ba7f..5f0e4a4 100644 --- a/target_snowflake/connector.py +++ b/target_snowflake/connector.py @@ -38,34 +38,6 @@ def evaluate_typemaps(type_maps, compare_value, unmatched_value): return unmatched_value -def _jsonschema_type_check(jsonschema_type: dict, type_check: Tuple[str]) -> bool: - """Return True if the jsonschema_type supports the provided type. - - Args: - jsonschema_type: The type dict. - type_check: A tuple of type strings to look for. - - Returns: - True if the schema suports the type. - """ - if "type" in jsonschema_type: - if isinstance(jsonschema_type["type"], (list, tuple)): - for schema_type in jsonschema_type["type"]: - if schema_type in type_check: - return True - else: - if jsonschema_type.get("type") in type_check: # noqa: PLR5501 - return True - - # TODO: remove following release of https://github.com/meltano/sdk/issues/1774 - if any( - _jsonschema_type_check(t, type_check) - for t in jsonschema_type.get("anyOf", ()) - ): - return True - - return False - class SnowflakeConnector(SQLConnector): """Snowflake Target Connector. @@ -267,12 +239,12 @@ def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: TypeMap(eq, sqlalchemy.types.VARCHAR(maxlength), None), ] type_maps = [ - TypeMap(_jsonschema_type_check, NUMBER(), ("integer",)), - TypeMap(_jsonschema_type_check, VARIANT(), ("object",)), - TypeMap(_jsonschema_type_check, VARIANT(), ("array",)), + TypeMap(th._jsonschema_type_check, NUMBER(), ("integer",)), + TypeMap(th._jsonschema_type_check, VARIANT(), ("object",)), + TypeMap(th._jsonschema_type_check, VARIANT(), ("array",)), ] # apply type maps - if _jsonschema_type_check(jsonschema_type, ("string",)): + if th._jsonschema_type_check(jsonschema_type, ("string",)): datelike_type = th.get_datelike_property_type(jsonschema_type) target_type = evaluate_typemaps(string_submaps, datelike_type, target_type) else: diff --git a/target_snowflake/sinks.py b/target_snowflake/sinks.py index 4e8dba2..cc4a2e4 100644 --- a/target_snowflake/sinks.py +++ b/target_snowflake/sinks.py @@ -1,8 +1,6 @@ """Snowflake target sink class, which handles writing streams.""" from __future__ import annotations -import gzip -import json import os import typing as t from urllib.parse import urlparse @@ -14,7 +12,6 @@ BaseBatchFileEncoding, BatchConfig, BatchFileFormat, - StorageTarget, ) from singer_sdk.helpers._typing import conform_record_data_types from singer_sdk.sinks import SQLSink @@ -235,3 +232,15 @@ def process_batch_files( raise NotImplementedError( f"Unsupported batch file encoding: {encoding.format}" ) + + # TODO: remove after https://github.com/meltano/sdk/issues/1819 is fixed + def _singer_validate_message(self, record: dict) -> None: + """Ensure record conforms to Singer Spec. + + Args: + record: Record (after parsing, schema validations and transformations). + + Raises: + MissingKeyPropertiesError: If record is missing one or more key properties. + """ + pass diff --git a/target_snowflake/target.py b/target_snowflake/target.py index 2979657..7d6352d 100644 --- a/target_snowflake/target.py +++ b/target_snowflake/target.py @@ -69,53 +69,5 @@ class TargetSnowflake(SQLTarget): default_sink_class = SnowflakeSink - def get_sink( - self, - stream_name: str, - *, - record: dict | None = None, - schema: dict | None = None, - key_properties: list[str] | None = None, - ) -> Sink: - _ = record # Custom implementations may use record in sink selection. - if schema is None: - self._assert_sink_exists(stream_name) - return self._sinks_active[stream_name] - - existing_sink = self._sinks_active.get(stream_name, None) - if not existing_sink: - return self.add_sink(stream_name, schema, key_properties) - - # Diffing was not accounting for metadata columns added by the target. - # The existing schema has the columns but the original from the SCHEMA - # message does not. - clean_existing_schema = existing_sink.schema.copy() - if self.config.get("add_record_metadata", True): - existing_props_copy = existing_sink.schema["properties"].copy() - for col in { - "_sdc_extracted_at", - "_sdc_received_at", - "_sdc_batched_at", - "_sdc_deleted_at", - "_sdc_sequence", - "_sdc_table_version", - }: - existing_props_copy.pop(col, None) - clean_existing_schema["properties"] = existing_props_copy - if ( - clean_existing_schema != schema - or existing_sink.key_properties != key_properties - ): - self.logger.info( - "Schema or key properties for '%s' stream have changed. " - "Initializing a new '%s' sink...", - stream_name, - stream_name, - ) - self._sinks_to_clear.append(self._sinks_active.pop(stream_name)) - return self.add_sink(stream_name, schema, key_properties) - - return existing_sink - if __name__ == "__main__": TargetSnowflake.cli() diff --git a/tests/test_target_snowflake.py b/tests/test_target_snowflake.py index 6ec3ec5..5c55ac0 100644 --- a/tests/test_target_snowflake.py +++ b/tests/test_target_snowflake.py @@ -8,7 +8,7 @@ from typing import Any import pytest -from singer_sdk.testing import TargetTestRunner, get_test_class +from singer_sdk.testing import TargetTestRunner, get_target_test_class from target_snowflake.target import TargetSnowflake @@ -26,29 +26,16 @@ } -# TODO: replace when upstream issue resolves -# https://github.com/meltano/sdk/pull/1752 -class CustomRunner(TargetTestRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def sync_all(self, *args, **kwargs): - try: - super().sync_all(*args, **kwargs) - finally: - self.target_input = None - - class BaseSnowflakeTargetTests: """Base class for Snowflake target tests.""" - @pytest.fixture(scope="class") + @pytest.fixture() def connection(self, runner): return runner.singer_class.default_sink_class.connector_class( runner.config ).connection - @pytest.fixture(scope="class") + @pytest.fixture() def resource(self, runner, connection): # noqa: ANN201 """Generic external resource. @@ -72,13 +59,12 @@ def resource(self, runner, connection): # noqa: ANN201 STANDARD_TEST_CONFIG[ "default_target_schema" ] = f"TARGET_SNOWFLAKE_{uuid.uuid4().hex[0:6]!s}" -StandardTargetTests = get_test_class( - test_runner=CustomRunner( - target_class=TargetSnowflake, - config=STANDARD_TEST_CONFIG, - ), - test_suites=[target_tests], +StandardTargetTests = get_target_test_class( + target_class=TargetSnowflake, + config=STANDARD_TEST_CONFIG, + custom_suites=[target_tests], suite_config=None, + include_target_tests=False, ) @@ -92,13 +78,12 @@ class TestTargetSnowflake(BaseSnowflakeTargetTests, StandardTargetTests): # typ "default_target_schema" ] = f"TARGET_SNOWFLAKE_{uuid.uuid4().hex[0:6]!s}" BATCH_TEST_CONFIG["add_record_metadata"] = False -BatchTargetTests = get_test_class( - test_runner=CustomRunner( - target_class=TargetSnowflake, - config=BATCH_TEST_CONFIG, - ), - test_suites=[batch_target_tests], +BatchTargetTests = get_target_test_class( + target_class=TargetSnowflake, + config=BATCH_TEST_CONFIG, + custom_suites=[batch_target_tests], suite_config=None, + include_target_tests=False, )