From 325cd7d36a0939286c58f452e87de57658ca36dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Fri, 14 Oct 2022 16:12:02 -0500 Subject: [PATCH 1/2] docs: Document an example implementation and usage of `BaseHATEOASPaginator` --- singer_sdk/pagination.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/singer_sdk/pagination.py b/singer_sdk/pagination.py index 8e294c3a5..12863d7ac 100644 --- a/singer_sdk/pagination.py +++ b/singer_sdk/pagination.py @@ -180,6 +180,37 @@ class BaseHATEOASPaginator(BaseAPIPaginator[Optional[ParseResult]], metaclass=AB This paginator expects responses to have a key "next" with a value like "https://api.com/link/to/next-item". + + The :attr:`~singer_sdk.pagination.BaseAPIPaginator.current_value` attribute of + this paginator is a `urllib.parse.ParseResult`_ object. This object + contains the following attributes: + + - scheme + - netloc + - path + - params + - query + - fragment + + That means you can access the parse the query params in your stream like this: + + .. code-block:: python + + class MyHATEOASPaginator(BaseHATEOASPaginator): + def get_next_url(self, response): + return response.json().get("next") + + class MyStream(Stream): + def get_new_paginator(self): + return MyHATEOASPaginator() + + def get_url_params(self, next_page_token) -> dict: + if next_page_token: + return dict(parse_qsl(next_page_token.query)) + return {} + + .. _`urllib.parse.ParseResult`: + https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse """ def __init__(self, *args: Any, **kwargs: Any) -> None: From 87f1e3dace261ba491aaa35361ccc35e591c73da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Fri, 14 Oct 2022 20:08:58 -0500 Subject: [PATCH 2/2] Fix type --- singer_sdk/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/pagination.py b/singer_sdk/pagination.py index 12863d7ac..4af95557f 100644 --- a/singer_sdk/pagination.py +++ b/singer_sdk/pagination.py @@ -192,7 +192,7 @@ class BaseHATEOASPaginator(BaseAPIPaginator[Optional[ParseResult]], metaclass=AB - query - fragment - That means you can access the parse the query params in your stream like this: + That means you can access and parse the query params in your stream like this: .. code-block:: python