diff --git a/docs/batch.md b/docs/batch.md index df6ef11786..45b1fc1f1b 100644 --- a/docs/batch.md +++ b/docs/batch.md @@ -52,7 +52,7 @@ AWS S3 ### `encoding` -The `encoding` field is used to specify the format and compression of the batch files. Currently only `jsonl` and `gzip` are supported, respectively. +The `encoding` field is used to specify the format and compression of the batch files. Currently `jsonl`, `gzip` and `parquet` are supported. ### `manifest` diff --git a/docs/conf.py b/docs/conf.py index 519b2af0c3..92f3775887 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,6 +39,7 @@ "sphinx.ext.napoleon", "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", "sphinx_copybutton", "myst_parser", "sphinx_reredirects", @@ -127,3 +128,10 @@ redirects = { "porting.html": "guides/porting.html", } + +# -- Options for intersphinx ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration +intersphinx_mapping = { + "requests": ("https://requests.readthedocs.io/en/latest/", None), + "python": ("https://docs.python.org/3/", None), +} diff --git a/docs/dev_guide.md b/docs/dev_guide.md index 75dcf60dcf..888ff03faf 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -177,6 +177,14 @@ Some APIs instead return the records as values inside an object where each key i ] ``` +## Extra features + +The following [extra features](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#extras) are available for the Singer SDK: + +- `s3` - Enables AWS S3 as a [BATCH storage](batch.md#the-batch-message). +- `parquet` - Enables as [BATCH encoding](batch.md#encoding). +- `testing` - Pytest dependencies required to use the [Tap & Target Testing Frameworks](testing.md). + ## Resources ### Detailed Class Reference diff --git a/noxfile.py b/noxfile.py index cd36003ba6..ea8e05ee9f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -58,7 +58,7 @@ def mypy(session: Session) -> None: """Check types with mypy.""" args = session.posargs or ["singer_sdk"] - session.install(".[s3,testing,parquet]") + session.install(".[crypto,parquet,s3,testing]") session.install( "exceptiongroup", "mypy", @@ -79,7 +79,7 @@ def mypy(session: Session) -> None: @session(python=python_versions) def tests(session: Session) -> None: """Execute pytest tests and compute coverage.""" - session.install(".[s3,parquet]") + session.install(".[crypto,parquet,s3]") session.install(*test_dependencies) sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") @@ -112,7 +112,7 @@ def tests(session: Session) -> None: @session(python=main_python_version) def benches(session: Session) -> None: """Run benchmarks.""" - session.install(".[s3]") + session.install(".[crypto,s3]") session.install(*test_dependencies) sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") if sqlalchemy_version: @@ -134,7 +134,7 @@ def update_snapshots(session: Session) -> None: """Update pytest snapshots.""" args = session.posargs or ["-m", "snapshot"] - session.install(".") + session.install(".[crypto]") session.install(*test_dependencies) session.run("pytest", "--snapshot-update", *args) diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index d31ab1deeb..82e0556bc3 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -131,13 +131,10 @@ def authenticate_request( """Authenticate a request. Args: - request: A `request object`_. + request: A :class:`requests.PreparedRequest` object. Returns: The authenticated request object. - - .. _request object: - https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest """ request.headers.update(self.auth_headers) @@ -154,13 +151,10 @@ def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: and returns the result. Args: - r: A `request object`_. + r: A :class:`requests.PreparedRequest` object. Returns: The authenticated request object. - - .. _request object: - https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest """ return self.authenticate_request(r) diff --git a/singer_sdk/pagination.py b/singer_sdk/pagination.py index 51ce52489c..82e02d6b77 100644 --- a/singer_sdk/pagination.py +++ b/singer_sdk/pagination.py @@ -181,7 +181,7 @@ class BaseHATEOASPaginator( like "https://api.com/link/to/next-item". The :attr:`~singer_sdk.pagination.BaseAPIPaginator.current_value` attribute of - this paginator is a `urllib.parse.ParseResult`_ object. This object + this paginator is a :class:`urllib.parse.ParseResult` object. This object contains the following attributes: - scheme @@ -208,9 +208,6 @@ def get_url_params(self, next_page_token) -> dict: if next_page_token: return dict(parse_qsl(next_page_token.query)) return {} - - .. _`urllib.parse.ParseResult`: - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse """ def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index fc1ce94150..1c2f46c97c 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -403,9 +403,7 @@ def print_version( Args: print_fn: A function to use to display the plugin version. - Defaults to `print`_. - - .. _print: https://docs.python.org/3/library/functions.html#print + Defaults to :py:func:`print`. """ print_fn(f"{cls.name} v{cls.plugin_version}, Meltano SDK v{cls.sdk_version}") diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index a9cfa568af..87e1001d9a 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -137,10 +137,7 @@ def requests_session(self) -> requests.Session: """Get requests session. Returns: - The `requests.Session`_ object for HTTP requests. - - .. _requests.Session: - https://requests.readthedocs.io/en/latest/api.html#requests.Session + The :class:`requests.Session` object for HTTP requests. """ if not self._requests_session: self._requests_session = requests.Session() @@ -168,14 +165,11 @@ def validate_response(self, response: requests.Response) -> None: .. image:: ../images/200.png Args: - response: A `requests.Response`_ object. + response: A :class:`requests.Response` object. Raises: FatalAPIError: If the request is not retriable. RetriableAPIError: If the request is retriable. - - .. _requests.Response: - https://requests.readthedocs.io/en/latest/api.html#requests.Response """ if ( response.status_code in self.extra_retry_statuses @@ -198,7 +192,7 @@ def response_error_message(self, response: requests.Response) -> str: WARNING - Override this method when the URL path may contain secrets or PII Args: - response: A `requests.Response`_ object. + response: A :class:`requests.Response` object. Returns: str: The error message @@ -286,7 +280,7 @@ def get_url_params( If your source needs special handling and, for example, parentheses should not be encoded, you can return a string constructed with - `urllib.parse.urlencode`_: + :py:func:`urllib.parse.urlencode`: .. code-block:: python @@ -306,9 +300,6 @@ def get_url_params(self, context, next_page_token): Returns: Dictionary or encoded string with URL query parameters to use in the request. - - .. _urllib.parse.urlencode: - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode """ return {} @@ -322,16 +313,11 @@ def build_prepared_request( Uses the authenticator instance to mutate the request with authentication. Args: - *args: Arguments to pass to `requests.Request`_. - **kwargs: Keyword arguments to pass to `requests.Request`_. + *args: Arguments to pass to :class:`requests.Request`. + **kwargs: Keyword arguments to pass to :class:`requests.Request`. Returns: - A `requests.PreparedRequest`_ object. - - .. _requests.PreparedRequest: - https://requests.readthedocs.io/en/latest/api.html#requests.PreparedRequest - .. _requests.Request: - https://requests.readthedocs.io/en/latest/api.html#requests.Request + A :class:`requests.PreparedRequest` object. """ request = requests.Request(*args, **kwargs) self.requests_session.auth = self.authenticator @@ -460,7 +446,7 @@ def update_sync_costs( Args: request: the Request object that was just called. - response: the `requests.Response` object + response: the :class:`requests.Response` object context: the context passed to the call Returns: @@ -498,7 +484,7 @@ def calculate_sync_cost( Args: request: the API Request object that was just called. - response: the `requests.Response` object + response: the :class:`requests.Response` object context: the context passed to the call Returns: @@ -596,13 +582,10 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]: """Parse the response and return an iterator of result records. Args: - response: A raw `requests.Response`_ object. + response: A raw :class:`requests.Response` Yields: One item for every item found in the response. - - .. _requests.Response: - https://requests.readthedocs.io/en/latest/api.html#requests.Response """ yield from extract_jsonpath(self.records_jsonpath, input=response.json())