From 263f1bf4706cc7c5da1c627d3f3979379887f671 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Thu, 24 Oct 2024 02:02:54 +0800 Subject: [PATCH 01/16] Allow URLs with paths as base_url in ClientSession --- aiohttp/client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 343d20436e7..1ffe0391c09 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -301,8 +301,8 @@ def __init__( self._base_url = URL(base_url) self._base_url_origin = self._base_url.origin() assert ( - self._base_url_origin == self._base_url - ), "Only absolute URLs without path part are supported" + self._base_url.absolute + ), "Only absolute URLs are supported" loop = asyncio.get_running_loop() @@ -415,8 +415,9 @@ def _build_url(self, str_or_url: StrOrURL) -> URL: if self._base_url is None: return url else: - assert not url.absolute and url.path.startswith("/") - return self._base_url.join(url) + assert not url.absolute + assert self._base_url.path.endswith("/") + return self._base_url.join(URL(url.path_qs.lstrip("/"))) async def _request( self, From 7cb6b9ee2437dd0fe93f77e32c2e50b08f8e4967 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Thu, 24 Oct 2024 02:11:53 +0800 Subject: [PATCH 02/16] add test for base_url with path --- tests/test_client_session.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index c7d907b0010..1ab5523b2c7 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1122,6 +1122,18 @@ async def test_requote_redirect_url_default_disable() -> None: URL("http://example.com/test"), id="base_url=URL('http://example.com') url='/test'", ), + pytest.param( + URL("http://example.com/test1/"), + "test2", + URL("http://example.com/test1/test2"), + id="base_url=URL('http://example.com/test1/') url='test2'", + ), + pytest.param( + URL("http://example.com/test1/"), + "/test2", + URL("http://example.com/test1/test2"), + id="base_url=URL('http://example.com/test1/') url='/test2'", + ), ], ) async def test_build_url_returns_expected_url( From 52a8e3db4b5e8d184a22b097afba4ffcf3d728eb Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Thu, 24 Oct 2024 02:25:18 +0800 Subject: [PATCH 03/16] move base_url check to __init__ --- aiohttp/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 1ffe0391c09..416e1990123 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -303,6 +303,8 @@ def __init__( assert ( self._base_url.absolute ), "Only absolute URLs are supported" + if self._base_url is not None: + assert self._base_url.path.endswith("/") loop = asyncio.get_running_loop() @@ -416,7 +418,6 @@ def _build_url(self, str_or_url: StrOrURL) -> URL: return url else: assert not url.absolute - assert self._base_url.path.endswith("/") return self._base_url.join(URL(url.path_qs.lstrip("/"))) async def _request( From 14b5c5a098e292cd627c04e1a79e33b312c2160e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 18:39:48 +0000 Subject: [PATCH 04/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 416e1990123..c91f5d3ff85 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -300,9 +300,7 @@ def __init__( else: self._base_url = URL(base_url) self._base_url_origin = self._base_url.origin() - assert ( - self._base_url.absolute - ), "Only absolute URLs are supported" + assert self._base_url.absolute, "Only absolute URLs are supported" if self._base_url is not None: assert self._base_url.path.endswith("/") From b022c81b924c4bc47e8785ab46cc1b57356c817d Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Thu, 24 Oct 2024 23:27:51 +0800 Subject: [PATCH 05/16] enhance base_url validation --- aiohttp/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index c91f5d3ff85..4888913e3c0 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -301,8 +301,8 @@ def __init__( self._base_url = URL(base_url) self._base_url_origin = self._base_url.origin() assert self._base_url.absolute, "Only absolute URLs are supported" - if self._base_url is not None: - assert self._base_url.path.endswith("/") + if self._base_url is not None and not self._base_url.path.endswith("/"): + raise ValueError("base_url must have a trailing '/'") loop = asyncio.get_running_loop() From 143276158ed4e0e87a1885df3226c1e8ab8a07c5 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Sun, 27 Oct 2024 21:18:05 +0800 Subject: [PATCH 06/16] revert behavior of url path joining --- aiohttp/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 4888913e3c0..183e9fc0ee5 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -416,7 +416,7 @@ def _build_url(self, str_or_url: StrOrURL) -> URL: return url else: assert not url.absolute - return self._base_url.join(URL(url.path_qs.lstrip("/"))) + return self._base_url.join(url) async def _request( self, From 39b5aa0d5ba83b2ffc52f474c2822d484c75ac1a Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Sun, 27 Oct 2024 21:20:34 +0800 Subject: [PATCH 07/16] update test for url join --- tests/test_client_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 1ab5523b2c7..d696d786392 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1131,7 +1131,7 @@ async def test_requote_redirect_url_default_disable() -> None: pytest.param( URL("http://example.com/test1/"), "/test2", - URL("http://example.com/test1/test2"), + URL("http://example.com/test2"), id="base_url=URL('http://example.com/test1/') url='/test2'", ), ], From d75f808731362d855b9b214fcdf74aeddf1b106f Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 01:54:55 +0800 Subject: [PATCH 08/16] update documentation --- CHANGES/9530.feature.rst | 2 ++ docs/client_reference.rst | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 CHANGES/9530.feature.rst diff --git a/CHANGES/9530.feature.rst b/CHANGES/9530.feature.rst new file mode 100644 index 00000000000..f5e61f8bcdf --- /dev/null +++ b/CHANGES/9530.feature.rst @@ -0,0 +1,2 @@ +Updated :py:class:`~aiohttp.ClientSession` to make ``base_url`` parameter allow url with path. +``base_url`` must have a path that end with a ``/`` -- by :user:`Cycloctane`. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 8cb6c91c2bc..973d79d90b7 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -62,9 +62,11 @@ The client session supports the context manager protocol for self closing. :param base_url: Base part of the URL (optional) - If set, it allows to skip the base part (https://docs.aiohttp.org) in - request calls. It must not include a path (as in - https://docs.aiohttp.org/en/stable). + + If set, it allows to prepend a base part to urls in request calls. It + must either have a path with a trailing ``/`` (as in + https://docs.aiohttp.org/en/stable/) or have no path at all (as in + https://docs.aiohttp.org). .. versionadded:: 3.8 @@ -372,6 +374,20 @@ The client session supports the context manager protocol for self closing. be encoded with :class:`~yarl.URL` (see :class:`~yarl.URL` to skip encoding). + If the ``_base_url`` parameter is set, request URL must be + relative and will be combined with ``_base_url`` to construct + the final request URL. See :meth:`~yarl.URL.join`. + + e.g.:: + + session = ClientSession(base_url="http://example.com/foo/") + + await session.request("GET", "bar") + # request for http://example.com/foo/bar + + await session.request("GET", "/bar") + # request for http://example.com/bar + :param params: Mapping, iterable of tuple of *key*/*value* pairs or string to be sent as parameters in the query string of the new request. Ignored for subsequent From ec59615288cc48f09bb7c0f11f4b448da82b5d27 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 08:52:09 +0800 Subject: [PATCH 09/16] add test for base_url without trailing slash --- tests/test_client_session.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index d696d786392..4378c3d3878 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1146,6 +1146,11 @@ async def test_build_url_returns_expected_url( assert session._build_url(url) == expected_url +async def test_base_url_without_trailing_slash(): + with pytest.raises(ValueError, match="base_url must have a trailing '/'"): + ClientSession(base_url="http://example.com/test") + + async def test_instantiation_with_invalid_timeout_value( loop: asyncio.AbstractEventLoop, ) -> None: From ef267b765f9cc0b536a6719590b6c9aa0f9828aa Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 08:55:04 +0800 Subject: [PATCH 10/16] fix return type of test_base_url_without_trailing_slash --- tests/test_client_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 4378c3d3878..73ca1ee7dd9 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1146,7 +1146,7 @@ async def test_build_url_returns_expected_url( assert session._build_url(url) == expected_url -async def test_base_url_without_trailing_slash(): +async def test_base_url_without_trailing_slash() -> None: with pytest.raises(ValueError, match="base_url must have a trailing '/'"): ClientSession(base_url="http://example.com/test") From ed44ebd2ba4ec625219578e237b1f99c20bf6ac3 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 09:44:11 +0800 Subject: [PATCH 11/16] add the explanation of path joining in documentation --- docs/client_reference.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 973d79d90b7..39156d8ee1d 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -376,7 +376,9 @@ The client session supports the context manager protocol for self closing. If the ``_base_url`` parameter is set, request URL must be relative and will be combined with ``_base_url`` to construct - the final request URL. See :meth:`~yarl.URL.join`. + the final request URL. The path in ``_base_url`` will be + ignored if request URL path starts with ``/``. + See :meth:`~yarl.URL.join`. e.g.:: From 99defe03b9a1767573bf34b1046d3ca1e817e385 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 09:45:23 +0800 Subject: [PATCH 12/16] add test param for url joining with query string and fragment --- tests/test_client_session.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 73ca1ee7dd9..9869a21d878 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1134,6 +1134,12 @@ async def test_requote_redirect_url_default_disable() -> None: URL("http://example.com/test2"), id="base_url=URL('http://example.com/test1/') url='/test2'", ), + pytest.param( + URL("http://example.com/test1/"), + "test2?q=foo#bar", + URL("http://example.com/test1/test2?q=foo#bar"), + id="base_url=URL('http://example.com/test1/') url='test2?q=foo#bar'", + ) ], ) async def test_build_url_returns_expected_url( From c9145315fe2fe5f5f8fb9c3549a09d206207f58d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 01:46:21 +0000 Subject: [PATCH 13/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_client_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 9869a21d878..54e8ff8c658 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -1139,7 +1139,7 @@ async def test_requote_redirect_url_default_disable() -> None: "test2?q=foo#bar", URL("http://example.com/test1/test2?q=foo#bar"), id="base_url=URL('http://example.com/test1/') url='test2?q=foo#bar'", - ) + ), ], ) async def test_build_url_returns_expected_url( From 09d8d30b39fd4949965593c2b74dc9816d6e7c7f Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Mon, 28 Oct 2024 23:26:50 +0800 Subject: [PATCH 14/16] apply suggestions on docs Co-authored-by: Sam Bull --- CHANGES/9530.feature.rst | 4 ++-- docs/client_reference.rst | 35 +++++++++++++++-------------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/CHANGES/9530.feature.rst b/CHANGES/9530.feature.rst index f5e61f8bcdf..cc4e75a13ca 100644 --- a/CHANGES/9530.feature.rst +++ b/CHANGES/9530.feature.rst @@ -1,2 +1,2 @@ -Updated :py:class:`~aiohttp.ClientSession` to make ``base_url`` parameter allow url with path. -``base_url`` must have a path that end with a ``/`` -- by :user:`Cycloctane`. +Updated :py:class:`~aiohttp.ClientSession` to support paths in ``base_url`` parameter. +``base_url`` paths must end with a ``/`` -- by :user:`Cycloctane`. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 39156d8ee1d..5b232ea8f9c 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -62,11 +62,22 @@ The client session supports the context manager protocol for self closing. :param base_url: Base part of the URL (optional) + If set, allows to join a base part to relative URLs in request calls. + If the URL has a path it must have a trailing ``/`` (as in + https://docs.aiohttp.org/en/stable/). - If set, it allows to prepend a base part to urls in request calls. It - must either have a path with a trailing ``/`` (as in - https://docs.aiohttp.org/en/stable/) or have no path at all (as in - https://docs.aiohttp.org). + Note that URL joining follows :rfc:`3986`. This means, in the most + common case the request URLs should have no leading slash: + + e.g.:: + + session = ClientSession(base_url="http://example.com/foo/") + + await session.request("GET", "bar") + # request for http://example.com/foo/bar + + await session.request("GET", "/bar") + # request for http://example.com/bar .. versionadded:: 3.8 @@ -374,22 +385,6 @@ The client session supports the context manager protocol for self closing. be encoded with :class:`~yarl.URL` (see :class:`~yarl.URL` to skip encoding). - If the ``_base_url`` parameter is set, request URL must be - relative and will be combined with ``_base_url`` to construct - the final request URL. The path in ``_base_url`` will be - ignored if request URL path starts with ``/``. - See :meth:`~yarl.URL.join`. - - e.g.:: - - session = ClientSession(base_url="http://example.com/foo/") - - await session.request("GET", "bar") - # request for http://example.com/foo/bar - - await session.request("GET", "/bar") - # request for http://example.com/bar - :param params: Mapping, iterable of tuple of *key*/*value* pairs or string to be sent as parameters in the query string of the new request. Ignored for subsequent From 85956f1177181f03719b7f78fc76e415a8c04822 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Oct 2024 09:38:10 -1000 Subject: [PATCH 15/16] cleanup whitespace --- docs/client_reference.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 5b232ea8f9c..8e809b7bcb3 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -69,15 +69,15 @@ The client session supports the context manager protocol for self closing. Note that URL joining follows :rfc:`3986`. This means, in the most common case the request URLs should have no leading slash: - e.g.:: + e.g.:: - session = ClientSession(base_url="http://example.com/foo/") + session = ClientSession(base_url="http://example.com/foo/") - await session.request("GET", "bar") - # request for http://example.com/foo/bar + await session.request("GET", "bar") + # request for http://example.com/foo/bar - await session.request("GET", "/bar") - # request for http://example.com/bar + await session.request("GET", "/bar") + # request for http://example.com/bar .. versionadded:: 3.8 From f8b0ed51b0d2818cb1519a82ad578d1d32ad2a66 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Oct 2024 09:49:21 -1000 Subject: [PATCH 16/16] try to improve formatting a bit more --- docs/client_reference.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 8e809b7bcb3..ed290277b50 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -67,17 +67,15 @@ The client session supports the context manager protocol for self closing. https://docs.aiohttp.org/en/stable/). Note that URL joining follows :rfc:`3986`. This means, in the most - common case the request URLs should have no leading slash: + common case the request URLs should have no leading slash, e.g.:: - e.g.:: + session = ClientSession(base_url="http://example.com/foo/") - session = ClientSession(base_url="http://example.com/foo/") + await session.request("GET", "bar") + # request for http://example.com/foo/bar - await session.request("GET", "bar") - # request for http://example.com/foo/bar - - await session.request("GET", "/bar") - # request for http://example.com/bar + await session.request("GET", "/bar") + # request for http://example.com/bar .. versionadded:: 3.8