From 16d8a1793e57b21f9f01b986a86903409362e2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 21 Dec 2023 20:58:51 -0600 Subject: [PATCH] refactor: Allow loading stream schemas from `importlib.resources.abc.Traversable` types --- .github/workflows/test.yml | 2 +- .../{{cookiecutter.library_name}}/rest-client.py | 10 ++++++++-- .../{{cookiecutter.library_name}}/streams.py | 4 ++-- samples/sample_tap_countries/countries_streams.py | 4 ++-- samples/sample_tap_gitlab/gitlab_graphql_streams.py | 5 ++--- samples/sample_tap_gitlab/gitlab_rest_streams.py | 4 ++-- samples/sample_tap_google_analytics/ga_tap_stream.py | 4 ++-- singer_sdk/helpers/_compat.py | 9 +++++++++ singer_sdk/streams/core.py | 7 ++++--- 9 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f1fef77e9..c60a2829aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ on: - ".github/workflows/test.yml" - ".github/workflows/constraints.txt" push: - branches: [main] + # branches: [main] paths: - "cookiecutter/**" - "samples/**" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py index dae2269dff..5b81a0dc28 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py @@ -5,34 +5,38 @@ {% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} import sys {% endif -%} -from pathlib import Path from typing import Any, Callable, Iterable import requests {% if cookiecutter.auth_method == "API Key" -%} from singer_sdk.authenticators import APIKeyAuthenticator +from singer_sdk.helpers._compat import resources from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream {% elif cookiecutter.auth_method == "Bearer Token" -%} from singer_sdk.authenticators import BearerTokenAuthenticator +from singer_sdk.helpers._compat import resources from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream {% elif cookiecutter.auth_method == "Basic Auth" -%} from singer_sdk.authenticators import BasicAuthenticator +from singer_sdk.helpers._compat import resources from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream {% elif cookiecutter.auth_method == "Custom or N/A" -%} +from singer_sdk.helpers._compat import resources from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream {% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} +from singer_sdk.helpers._compat import resources from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream @@ -50,7 +54,9 @@ {% endif -%} _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") + +# TODO: Delete this is if not using json files for schema definition +SCHEMAS_DIR = resources.files(__package__) / "schemas" class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py index 8272cbc24a..07a43d3f61 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py @@ -3,14 +3,14 @@ from __future__ import annotations import typing as t -from pathlib import Path from singer_sdk import typing as th # JSON Schema typing helpers +from singer_sdk.helpers._compat import resources from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream # TODO: Delete this is if not using json files for schema definition -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = resources.files(__package__) / "schemas" {%- if cookiecutter.stream_type == "GraphQL" %} diff --git a/samples/sample_tap_countries/countries_streams.py b/samples/sample_tap_countries/countries_streams.py index 708e1678a1..b8ad0840a5 100644 --- a/samples/sample_tap_countries/countries_streams.py +++ b/samples/sample_tap_countries/countries_streams.py @@ -9,12 +9,12 @@ from __future__ import annotations import abc -from pathlib import Path from singer_sdk import typing as th +from singer_sdk.helpers._compat import resources from singer_sdk.streams.graphql import GraphQLStream -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = resources.files(__package__) / "schemas" class CountriesAPIStream(GraphQLStream, metaclass=abc.ABCMeta): diff --git a/samples/sample_tap_gitlab/gitlab_graphql_streams.py b/samples/sample_tap_gitlab/gitlab_graphql_streams.py index b29fbc13ee..b824e0aae6 100644 --- a/samples/sample_tap_gitlab/gitlab_graphql_streams.py +++ b/samples/sample_tap_gitlab/gitlab_graphql_streams.py @@ -6,13 +6,12 @@ from __future__ import annotations -from pathlib import Path - +from singer_sdk.helpers._compat import resources from singer_sdk.streams import GraphQLStream SITE_URL = "https://gitlab.com/graphql" -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = resources.files(__package__) / "schemas" class GitlabGraphQLStream(GraphQLStream): diff --git a/samples/sample_tap_gitlab/gitlab_rest_streams.py b/samples/sample_tap_gitlab/gitlab_rest_streams.py index 1480a017d6..43c6843bde 100644 --- a/samples/sample_tap_gitlab/gitlab_rest_streams.py +++ b/samples/sample_tap_gitlab/gitlab_rest_streams.py @@ -3,9 +3,9 @@ from __future__ import annotations import typing as t -from pathlib import Path from singer_sdk.authenticators import SimpleAuthenticator +from singer_sdk.helpers._compat import resources from singer_sdk.pagination import SimpleHeaderPaginator from singer_sdk.streams.rest import RESTStream from singer_sdk.typing import ( @@ -17,7 +17,7 @@ StringType, ) -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = resources.files(__package__) / "schemas" DEFAULT_URL_BASE = "https://gitlab.com/api/v4" diff --git a/samples/sample_tap_google_analytics/ga_tap_stream.py b/samples/sample_tap_google_analytics/ga_tap_stream.py index 04c3a253eb..d80f367aa1 100644 --- a/samples/sample_tap_google_analytics/ga_tap_stream.py +++ b/samples/sample_tap_google_analytics/ga_tap_stream.py @@ -4,14 +4,14 @@ import datetime import typing as t -from pathlib import Path from singer_sdk.authenticators import OAuthJWTAuthenticator +from singer_sdk.helpers._compat import resources from singer_sdk.streams import RESTStream GOOGLE_OAUTH_ENDPOINT = "https://oauth2.googleapis.com/token" GA_OAUTH_SCOPES = "https://www.googleapis.com/auth/analytics.readonly" -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = resources.files(__package__) / "schemas" class GoogleJWTAuthenticator(OAuthJWTAuthenticator): diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py index cc84576a82..0959464657 100644 --- a/singer_sdk/helpers/_compat.py +++ b/singer_sdk/helpers/_compat.py @@ -22,6 +22,14 @@ else: from importlib import resources +if sys.version_info < (3, 9): + from importlib_resources.abc import Traversable +elif sys.version_info < (3, 12): + from importlib.abc import Traversable +else: + from importlib.resources.abc import Traversable + + if sys.version_info < (3, 11): from backports.datetime_fromisoformat import MonkeyPatch @@ -35,6 +43,7 @@ "metadata", "final", "resources", + "Traversable", "entry_points", "datetime_fromisoformat", "date_fromisoformat", diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index afcc1c016b..459b9e7615 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -54,6 +54,7 @@ if t.TYPE_CHECKING: import logging + from singer_sdk.helpers._compat import Traversable from singer_sdk.tap_base import Tap # Replication methods @@ -136,7 +137,7 @@ def __init__( self._replication_key: str | None = None self._primary_keys: t.Sequence[str] | None = None self._state_partitioning_keys: list[str] | None = None - self._schema_filepath: Path | None = None + self._schema_filepath: Path | Traversable | None = None self._metadata: singer.MetadataMapping | None = None self._mask: singer.SelectionMask | None = None self._schema: dict @@ -160,7 +161,7 @@ def __init__( raise ValueError(msg) if self.schema_filepath: - self._schema = json.loads(Path(self.schema_filepath).read_text()) + self._schema = json.loads(self.schema_filepath.read_text()) if not self.schema: msg = ( @@ -421,7 +422,7 @@ def get_replication_key_signpost( return utc_now() if self.is_timestamp_replication_key else None @property - def schema_filepath(self) -> Path | None: + def schema_filepath(self) -> Path | Traversable | None: """Get path to schema file. Returns: