From 7b172b045fd5f90d7775638345267c2816e75ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 21 Dec 2023 20:58:51 -0600 Subject: [PATCH] refactor: Allow loading stream schemas from `importlib.resources.abc.Traversable` types --- .../{{cookiecutter.tap_id}}/pyproject.toml | 1 + .../rest-client.py | 12 +++++++---- .../{{cookiecutter.library_name}}/streams.py | 10 ++++++++-- poetry.lock | 2 +- pyproject.toml | 2 +- samples/aapl/aapl.py | 9 +++++++-- .../sample_tap_countries/countries_streams.py | 4 ++-- .../gitlab_graphql_streams.py | 5 ++--- .../sample_tap_gitlab/gitlab_rest_streams.py | 4 ++-- .../ga_tap_stream.py | 4 ++-- singer_sdk/helpers/_compat.py | 20 +++++++++++++------ singer_sdk/streams/core.py | 7 ++++--- singer_sdk/testing/runners.py | 12 ++++++----- singer_sdk/testing/templates.py | 14 ++++++------- tests/samples/test_target_csv.py | 5 ++++- 15 files changed, 70 insertions(+), 41 deletions(-) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 8fc4f083f..772e4b906 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -21,6 +21,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8,<4" +importlib-resources = { version = "==6.1.*", python = "<3.9" } singer-sdk = { version="~=0.34.1" } fs-s3fs = { version = "~=1.1.1", optional = true } {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py index dae2269df..21316908b 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py @@ -2,10 +2,7 @@ from __future__ import annotations -{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} import sys -{% endif -%} -from pathlib import Path from typing import Any, Callable, Iterable import requests @@ -49,8 +46,15 @@ {% endif -%} +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") + +# TODO: Delete this is if not using json files for schema definition +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py index 8272cbc24..69c955e6f 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py @@ -2,15 +2,21 @@ from __future__ import annotations +import sys import typing as t -from pathlib import Path from singer_sdk import typing as th # JSON Schema typing helpers from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + + # TODO: Delete this is if not using json files for schema definition -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" {%- if cookiecutter.stream_type == "GraphQL" %} diff --git a/poetry.lock b/poetry.lock index 4a1ddef8f..d9d49f97c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3046,4 +3046,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = ">=3.7.1" -content-hash = "bd5ad4eb7d109f3d184dcfb99e6808e6209776dfc741b86e3ba3ac3369d64449" +content-hash = "b31a1c8736111f0a95b99d84aec5444b73c6e11ffb4e7a59d15c490257f69b8a" diff --git a/pyproject.toml b/pyproject.toml index 34f3fca86..cb7965d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ click = "~=8.0" cryptography = ">=3.4.6" fs = ">=2.4.16" importlib-metadata = {version = "<7.0.0", python = "<3.12"} -importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""} +importlib-resources = {version = ">=5.12.0", python = "<3.9"} inflection = ">=0.5.1" joblib = ">=1.0.1" jsonpath-ng = ">=1.5.3" diff --git a/samples/aapl/aapl.py b/samples/aapl/aapl.py index ecbf032de..7784a34f2 100644 --- a/samples/aapl/aapl.py +++ b/samples/aapl/aapl.py @@ -3,11 +3,16 @@ from __future__ import annotations import json -from pathlib import Path +import sys from singer_sdk import Stream, Tap -PROJECT_DIR = Path(__file__).parent +if sys.version_info < (3, 9): + import importlib_resources +else: + import importlib.resources as importlib_resources + +PROJECT_DIR = importlib_resources.files("samples.aapl") class AAPL(Stream): diff --git a/samples/sample_tap_countries/countries_streams.py b/samples/sample_tap_countries/countries_streams.py index 708e1678a..3b68a5571 100644 --- a/samples/sample_tap_countries/countries_streams.py +++ b/samples/sample_tap_countries/countries_streams.py @@ -9,12 +9,12 @@ from __future__ import annotations import abc -from pathlib import Path from singer_sdk import typing as th +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.streams.graphql import GraphQLStream -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" class CountriesAPIStream(GraphQLStream, metaclass=abc.ABCMeta): diff --git a/samples/sample_tap_gitlab/gitlab_graphql_streams.py b/samples/sample_tap_gitlab/gitlab_graphql_streams.py index b29fbc13e..303964615 100644 --- a/samples/sample_tap_gitlab/gitlab_graphql_streams.py +++ b/samples/sample_tap_gitlab/gitlab_graphql_streams.py @@ -6,13 +6,12 @@ from __future__ import annotations -from pathlib import Path - +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.streams import GraphQLStream SITE_URL = "https://gitlab.com/graphql" -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" class GitlabGraphQLStream(GraphQLStream): diff --git a/samples/sample_tap_gitlab/gitlab_rest_streams.py b/samples/sample_tap_gitlab/gitlab_rest_streams.py index 1480a017d..1db629099 100644 --- a/samples/sample_tap_gitlab/gitlab_rest_streams.py +++ b/samples/sample_tap_gitlab/gitlab_rest_streams.py @@ -3,9 +3,9 @@ from __future__ import annotations import typing as t -from pathlib import Path from singer_sdk.authenticators import SimpleAuthenticator +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.pagination import SimpleHeaderPaginator from singer_sdk.streams.rest import RESTStream from singer_sdk.typing import ( @@ -17,7 +17,7 @@ StringType, ) -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" DEFAULT_URL_BASE = "https://gitlab.com/api/v4" diff --git a/samples/sample_tap_google_analytics/ga_tap_stream.py b/samples/sample_tap_google_analytics/ga_tap_stream.py index 04c3a253e..5bd0503fb 100644 --- a/samples/sample_tap_google_analytics/ga_tap_stream.py +++ b/samples/sample_tap_google_analytics/ga_tap_stream.py @@ -4,14 +4,14 @@ import datetime import typing as t -from pathlib import Path from singer_sdk.authenticators import OAuthJWTAuthenticator +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.streams import RESTStream GOOGLE_OAUTH_ENDPOINT = "https://oauth2.googleapis.com/token" GA_OAUTH_SCOPES = "https://www.googleapis.com/auth/analytics.readonly" -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" class GoogleJWTAuthenticator(OAuthJWTAuthenticator): diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py index cc84576a8..c9a7df6cc 100644 --- a/singer_sdk/helpers/_compat.py +++ b/singer_sdk/helpers/_compat.py @@ -12,15 +12,22 @@ from importlib import metadata from typing import final # noqa: ICN003 -if sys.version_info < (3, 12): - from importlib_metadata import entry_points +if sys.version_info < (3, 9): + import importlib_resources else: - from importlib.metadata import entry_points + from importlib import resources as importlib_resources if sys.version_info < (3, 9): - import importlib_resources as resources + from importlib_resources.abc import Traversable +elif sys.version_info < (3, 12): + from importlib.abc import Traversable else: - from importlib import resources + from importlib.resources.abc import Traversable + +if sys.version_info < (3, 12): + from importlib_metadata import entry_points +else: + from importlib.metadata import entry_points if sys.version_info < (3, 11): from backports.datetime_fromisoformat import MonkeyPatch @@ -34,9 +41,10 @@ __all__ = [ "metadata", "final", - "resources", "entry_points", "datetime_fromisoformat", "date_fromisoformat", "time_fromisoformat", + "importlib_resources", + "Traversable", ] diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index afcc1c016..459b9e761 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -54,6 +54,7 @@ if t.TYPE_CHECKING: import logging + from singer_sdk.helpers._compat import Traversable from singer_sdk.tap_base import Tap # Replication methods @@ -136,7 +137,7 @@ def __init__( self._replication_key: str | None = None self._primary_keys: t.Sequence[str] | None = None self._state_partitioning_keys: list[str] | None = None - self._schema_filepath: Path | None = None + self._schema_filepath: Path | Traversable | None = None self._metadata: singer.MetadataMapping | None = None self._mask: singer.SelectionMask | None = None self._schema: dict @@ -160,7 +161,7 @@ def __init__( raise ValueError(msg) if self.schema_filepath: - self._schema = json.loads(Path(self.schema_filepath).read_text()) + self._schema = json.loads(self.schema_filepath.read_text()) if not self.schema: msg = ( @@ -421,7 +422,7 @@ def get_replication_key_signpost( return utc_now() if self.is_timestamp_replication_key else None @property - def schema_filepath(self) -> Path | None: + def schema_filepath(self) -> Path | Traversable | None: """Get path to schema file. Returns: diff --git a/singer_sdk/testing/runners.py b/singer_sdk/testing/runners.py index f6e135fe8..c5ec10a10 100644 --- a/singer_sdk/testing/runners.py +++ b/singer_sdk/testing/runners.py @@ -8,11 +8,15 @@ import typing as t from collections import defaultdict from contextlib import redirect_stderr, redirect_stdout -from pathlib import Path from singer_sdk import Tap, Target from singer_sdk.testing.config import SuiteConfig +if t.TYPE_CHECKING: + from pathlib import Path + + from singer_sdk.helpers._compat import Traversable + class SingerTestRunner(metaclass=abc.ABCMeta): """Base Singer Test Runner.""" @@ -197,7 +201,7 @@ def __init__( target_class: type[Target], config: dict | None = None, suite_config: SuiteConfig | None = None, - input_filepath: Path | None = None, + input_filepath: Path | Traversable | None = None, input_io: io.StringIO | None = None, **kwargs: t.Any, ) -> None: @@ -242,9 +246,7 @@ def target_input(self) -> t.IO[str]: if self.input_io: self._input = self.input_io elif self.input_filepath: - self._input = Path(self.input_filepath).open( # noqa: SIM115 - encoding="utf8", - ) + self._input = self.input_filepath.open(encoding="utf8") return t.cast(t.IO[str], self._input) @target_input.setter diff --git a/singer_sdk/testing/templates.py b/singer_sdk/testing/templates.py index 0f01e3f49..4a16feb05 100644 --- a/singer_sdk/testing/templates.py +++ b/singer_sdk/testing/templates.py @@ -5,12 +5,12 @@ import contextlib import typing as t import warnings -from pathlib import Path -from singer_sdk.helpers._compat import resources +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.testing import target_test_streams if t.TYPE_CHECKING: + from singer_sdk.helpers._compat import Traversable from singer_sdk.streams import Stream from .config import SuiteConfig @@ -322,14 +322,14 @@ def run( # type: ignore[override] """ # get input from file if getattr(self, "singer_filepath", None): - assert Path( - self.singer_filepath, - ).exists(), f"Singer file {self.singer_filepath} does not exist." + assert ( + self.singer_filepath.is_file() + ), f"Singer file {self.singer_filepath} does not exist." runner.input_filepath = self.singer_filepath super().run(config, resource, runner) @property - def singer_filepath(self) -> Path: + def singer_filepath(self) -> Traversable: """Get path to singer JSONL formatted messages file. Files will be sourced from `./target_test_streams/.singer`. @@ -337,4 +337,4 @@ def singer_filepath(self) -> Path: Returns: The expected Path to this tests singer file. """ - return resources.files(target_test_streams).joinpath(f"{self.name}.singer") # type: ignore[no-any-return] + return importlib_resources.files(target_test_streams) / f"{self.name}.singer" # type: ignore[no-any-return] diff --git a/tests/samples/test_target_csv.py b/tests/samples/test_target_csv.py index e55aa3cbc..a0a3f3497 100644 --- a/tests/samples/test_target_csv.py +++ b/tests/samples/test_target_csv.py @@ -15,6 +15,7 @@ from samples.sample_mapper.mapper import StreamTransform from samples.sample_tap_countries.countries_tap import SampleTapCountries from samples.sample_target_csv.csv_target import SampleTargetCSV +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.testing import ( get_target_test_class, sync_end_to_end, @@ -146,7 +147,9 @@ def test_target_batching(): } -SAMPLE_FILENAME = Path(__file__).parent / Path("./resources/messages.jsonl") +SAMPLE_FILENAME = ( + importlib_resources.files("tests.samples") / "resources/messages.jsonl" +) EXPECTED_OUTPUT = """"id" "name" 1 "Chris" 2 "Mike"