refactor: Allow loading stream schemas from `importlib.resources.abc.…

…Traversable` types

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -21,6 +21,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4"
+importlib-resources = { version = "==6.1.*", python = "<3.9" }
 singer-sdk = { version="~=0.34.1" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py

@@ -2,10 +2,7 @@
 
 from __future__ import annotations
 
-{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%}
 import sys
-{% endif -%}
-from pathlib import Path
 from typing import Any, Callable, Iterable
 
 import requests
@@ -49,8 +46,15 @@
 
 {% endif -%}
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
 _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest]
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+
+# TODO: Delete this is if not using json files for schema definition
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py

@@ -2,15 +2,21 @@
 
 from __future__ import annotations
 
+import sys
 import typing as t
-from pathlib import Path
 
 from singer_sdk import typing as th  # JSON Schema typing helpers
 
 from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
+
 # TODO: Delete this is if not using json files for schema definition
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 {%- if cookiecutter.stream_type == "GraphQL" %}

pyproject.toml

@@ -46,7 +46,7 @@ click = "~=8.0"
 cryptography = ">=3.4.6"
 fs = ">=2.4.16"
 importlib-metadata = {version = "<7.0.0", python = "<3.12"}
-importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""}
+importlib-resources = {version = ">=5.12.0", python = "<3.9"}
 inflection = ">=0.5.1"
 joblib = ">=1.0.1"
 jsonpath-ng = ">=1.5.3"

samples/aapl/aapl.py

@@ -3,11 +3,16 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
+import sys
 
 from singer_sdk import Stream, Tap
 
-PROJECT_DIR = Path(__file__).parent
+if sys.version_info < (3, 9):
+    import importlib_resources
+else:
+    import importlib.resources as importlib_resources
+
+PROJECT_DIR = importlib_resources.files("samples.aapl")
 
 
 class AAPL(Stream):

samples/sample_tap_countries/countries_streams.py

@@ -9,12 +9,12 @@
 from __future__ import annotations
 
 import abc
-from pathlib import Path
 
 from singer_sdk import typing as th
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams.graphql import GraphQLStream
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class CountriesAPIStream(GraphQLStream, metaclass=abc.ABCMeta):

samples/sample_tap_gitlab/gitlab_graphql_streams.py

@@ -6,13 +6,12 @@
 
 from __future__ import annotations
 
-from pathlib import Path
-
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams import GraphQLStream
 
 SITE_URL = "https://gitlab.com/graphql"
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class GitlabGraphQLStream(GraphQLStream):

samples/sample_tap_gitlab/gitlab_rest_streams.py

@@ -3,9 +3,9 @@
 from __future__ import annotations
 
 import typing as t
-from pathlib import Path
 
 from singer_sdk.authenticators import SimpleAuthenticator
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.pagination import SimpleHeaderPaginator
 from singer_sdk.streams.rest import RESTStream
 from singer_sdk.typing import (
@@ -17,7 +17,7 @@
     StringType,
 )
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 DEFAULT_URL_BASE = "https://gitlab.com/api/v4"
 

samples/sample_tap_google_analytics/ga_tap_stream.py

@@ -4,14 +4,14 @@
 
 import datetime
 import typing as t
-from pathlib import Path
 
 from singer_sdk.authenticators import OAuthJWTAuthenticator
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams import RESTStream
 
 GOOGLE_OAUTH_ENDPOINT = "https://oauth2.googleapis.com/token"
 GA_OAUTH_SCOPES = "https://www.googleapis.com/auth/analytics.readonly"
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class GoogleJWTAuthenticator(OAuthJWTAuthenticator):

singer_sdk/helpers/_compat.py

@@ -12,15 +12,22 @@
     from importlib import metadata
     from typing import final  # noqa: ICN003
 
-if sys.version_info < (3, 12):
-    from importlib_metadata import entry_points
+if sys.version_info < (3, 9):
+    import importlib_resources
 else:
-    from importlib.metadata import entry_points
+    from importlib import resources as importlib_resources
 
 if sys.version_info < (3, 9):
-    import importlib_resources as resources
+    from importlib_resources.abc import Traversable
+elif sys.version_info < (3, 12):
+    from importlib.abc import Traversable
 else:
-    from importlib import resources
+    from importlib.resources.abc import Traversable
+
+if sys.version_info < (3, 12):
+    from importlib_metadata import entry_points
+else:
+    from importlib.metadata import entry_points
 
 if sys.version_info < (3, 11):
     from backports.datetime_fromisoformat import MonkeyPatch
@@ -34,9 +41,10 @@
 __all__ = [
     "metadata",
     "final",
-    "resources",
     "entry_points",
     "datetime_fromisoformat",
     "date_fromisoformat",
     "time_fromisoformat",
+    "importlib_resources",
+    "Traversable",
 ]

singer_sdk/streams/core.py

@@ -54,6 +54,7 @@
 if t.TYPE_CHECKING:
     import logging
 
+    from singer_sdk.helpers._compat import Traversable
     from singer_sdk.tap_base import Tap
 
 # Replication methods
@@ -136,7 +137,7 @@ def __init__(
         self._replication_key: str | None = None
         self._primary_keys: t.Sequence[str] | None = None
         self._state_partitioning_keys: list[str] | None = None
-        self._schema_filepath: Path | None = None
+        self._schema_filepath: Path | Traversable | None = None
         self._metadata: singer.MetadataMapping | None = None
         self._mask: singer.SelectionMask | None = None
         self._schema: dict
@@ -160,7 +161,7 @@ def __init__(
                 raise ValueError(msg)
 
         if self.schema_filepath:
-            self._schema = json.loads(Path(self.schema_filepath).read_text())
+            self._schema = json.loads(self.schema_filepath.read_text())
 
         if not self.schema:
             msg = (
@@ -421,7 +422,7 @@ def get_replication_key_signpost(
         return utc_now() if self.is_timestamp_replication_key else None
 
     @property
-    def schema_filepath(self) -> Path | None:
+    def schema_filepath(self) -> Path | Traversable | None:
         """Get path to schema file.
 
         Returns:

singer_sdk/testing/runners.py

@@ -8,11 +8,15 @@
 import typing as t
 from collections import defaultdict
 from contextlib import redirect_stderr, redirect_stdout
-from pathlib import Path
 
 from singer_sdk import Tap, Target
 from singer_sdk.testing.config import SuiteConfig
 
+if t.TYPE_CHECKING:
+    from pathlib import Path
+
+    from singer_sdk.helpers._compat import Traversable
+
 
 class SingerTestRunner(metaclass=abc.ABCMeta):
     """Base Singer Test Runner."""
@@ -197,7 +201,7 @@ def __init__(
         target_class: type[Target],
         config: dict | None = None,
         suite_config: SuiteConfig | None = None,
-        input_filepath: Path | None = None,
+        input_filepath: Path | Traversable | None = None,
         input_io: io.StringIO | None = None,
         **kwargs: t.Any,
     ) -> None:
@@ -242,9 +246,7 @@ def target_input(self) -> t.IO[str]:
             if self.input_io:
                 self._input = self.input_io
             elif self.input_filepath:
-                self._input = Path(self.input_filepath).open(  # noqa: SIM115
-                    encoding="utf8",
-                )
+                self._input = self.input_filepath.open(encoding="utf8")
         return t.cast(t.IO[str], self._input)
 
     @target_input.setter

singer_sdk/testing/templates.py

@@ -5,12 +5,12 @@
 import contextlib
 import typing as t
 import warnings
-from pathlib import Path
 
-from singer_sdk.helpers._compat import resources
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.testing import target_test_streams
 
 if t.TYPE_CHECKING:
+    from singer_sdk.helpers._compat import Traversable
     from singer_sdk.streams import Stream
 
     from .config import SuiteConfig
@@ -322,19 +322,19 @@ def run(  # type: ignore[override]
         """
         # get input from file
         if getattr(self, "singer_filepath", None):
-            assert Path(
-                self.singer_filepath,
-            ).exists(), f"Singer file {self.singer_filepath} does not exist."
+            assert (
+                self.singer_filepath.is_file()
+            ), f"Singer file {self.singer_filepath} does not exist."
             runner.input_filepath = self.singer_filepath
         super().run(config, resource, runner)
 
     @property
-    def singer_filepath(self) -> Path:
+    def singer_filepath(self) -> Traversable:
         """Get path to singer JSONL formatted messages file.
 
         Files will be sourced from `./target_test_streams/<test name>.singer`.
 
         Returns:
             The expected Path to this tests singer file.
         """
-        return resources.files(target_test_streams).joinpath(f"{self.name}.singer")  # type: ignore[no-any-return]
+        return importlib_resources.files(target_test_streams) / f"{self.name}.singer"  # type: ignore[no-any-return]

tests/samples/test_target_csv.py

@@ -15,6 +15,7 @@
 from samples.sample_mapper.mapper import StreamTransform
 from samples.sample_tap_countries.countries_tap import SampleTapCountries
 from samples.sample_target_csv.csv_target import SampleTargetCSV
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.testing import (
     get_target_test_class,
     sync_end_to_end,
@@ -146,7 +147,9 @@ def test_target_batching():
     }
 
 
-SAMPLE_FILENAME = Path(__file__).parent / Path("./resources/messages.jsonl")
+SAMPLE_FILENAME = (
+    importlib_resources.files("tests.samples") / "resources/messages.jsonl"
+)
 EXPECTED_OUTPUT = """"id"	"name"
 1	"Chris"
 2	"Mike"