From 6da04694fd87a39af9c3856048c9ff23ca815f88 Mon Sep 17 00:00:00 2001 From: Yuvi Panda Date: Fri, 17 Mar 2023 08:01:00 +0530 Subject: [PATCH] Support using netrc for non-proxy HTTP credentials Currently, setting `trust_env=True` in `ClientSession` will only use the `.netrc` file if present for authentication to proxies. With this change, the credentials will also be used for regular HTTP hosts, and Basic Auth will be sent if an appropriate entry is present in the user's `.netrc` file. PR #7131. Co-authored-by: Sam Bull Co-authored-by: Sviatoslav Sydorenko --- CHANGES/7131.feature | 1 + CONTRIBUTORS.txt | 1 + aiohttp/client.py | 1 + aiohttp/client_reqrep.py | 12 +++++-- aiohttp/helpers.py | 43 ++++++++++++++++++------ docs/client_reference.rst | 9 ++++++ docs/glossary.rst | 15 +++++++++ tests/conftest.py | 22 +++++++++++++ tests/test_client_request.py | 52 +++++++++++++++++++++++++++-- tests/test_helpers.py | 63 ++++++++++++++++++++++++++++++++++++ 10 files changed, 205 insertions(+), 14 deletions(-) create mode 100644 CHANGES/7131.feature diff --git a/CHANGES/7131.feature b/CHANGES/7131.feature new file mode 100644 index 00000000000..bd77aff3613 --- /dev/null +++ b/CHANGES/7131.feature @@ -0,0 +1 @@ +Added support for using Basic Auth credentials from :file:`.netrc` file when making HTTP requests with the :py:class:`~aiohttp.ClientSession` ``trust_env`` argument is set to ``True`` -- by :user:`yuvipanda`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a934dee9967..28359ea100f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -355,6 +355,7 @@ Yury Pliner Yury Selivanov Yusuke Tsutsumi Yuval Ofir +Yuvi Panda Zainab Lawal Zeal Wierslee Zlatan Sičanica diff --git a/aiohttp/client.py b/aiohttp/client.py index 565cc0ea561..e1befbeb68e 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -481,6 +481,7 @@ async def _request( ssl=ssl, proxy_headers=proxy_headers, traces=traces, + trust_env=self.trust_env, ) # connection timeout diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index dadc4e2fa7b..edaf7087fc5 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -1,5 +1,6 @@ import asyncio import codecs +import contextlib import dataclasses import functools import io @@ -45,7 +46,9 @@ BasicAuth, HeadersMixin, TimerNoop, + basicauth_from_netrc, is_expected_content_type, + netrc_from_env, noop, parse_mimetype, reify, @@ -210,6 +213,7 @@ def __init__( ssl: Union[SSLContext, bool, Fingerprint, None] = None, proxy_headers: Optional[LooseHeaders] = None, traces: Optional[List["Trace"]] = None, + trust_env: bool = False, ): match = _CONTAINS_CONTROL_CHAR_RE.search(method) if match: @@ -251,7 +255,7 @@ def __init__( self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) - self.update_auth(auth) + self.update_auth(auth, trust_env) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) @@ -428,10 +432,14 @@ def update_transfer_encoding(self) -> None: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) - def update_auth(self, auth: Optional[BasicAuth]) -> None: + def update_auth(self, auth: Optional[BasicAuth], trust_env: bool = False) -> None: """Set basic auth.""" if auth is None: auth = self.auth + if auth is None and trust_env and self.url.host is not None: + netrc_obj = netrc_from_env() + with contextlib.suppress(LookupError): + auth = basicauth_from_netrc(netrc_obj, self.url.host) if auth is None: return diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index f3a9026393e..431bd67dd14 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -41,7 +41,6 @@ Type, TypeVar, Union, - cast, overload, ) from urllib.parse import quote @@ -244,6 +243,35 @@ class ProxyInfo: proxy_auth: Optional[BasicAuth] +def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth: + """ + Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``. + + :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no + entry is found for the ``host``. + """ + if netrc_obj is None: + raise LookupError("No .netrc file found") + auth_from_netrc = netrc_obj.authenticators(host) + + if auth_from_netrc is None: + raise LookupError(f"No entry for {host!s} found in the `.netrc` file.") + login, account, password = auth_from_netrc + + # TODO(PY311): username = login or account + # Up to python 3.10, account could be None if not specified, + # and login will be empty string if not specified. From 3.11, + # login and account will be empty string if not specified. + username = login if (login or account is None) else account + + # TODO(PY311): Remove this, as password will be empty string + # if not specified + if password is None: + password = "" + + return BasicAuth(username, password) + + def proxies_from_env() -> Dict[str, ProxyInfo]: proxy_urls = { k: URL(v) @@ -261,16 +289,11 @@ def proxies_from_env() -> Dict[str, ProxyInfo]: ) continue if netrc_obj and auth is None: - auth_from_netrc = None if proxy.host is not None: - auth_from_netrc = netrc_obj.authenticators(proxy.host) - if auth_from_netrc is not None: - # auth_from_netrc is a (`user`, `account`, `password`) tuple, - # `user` and `account` both can be username, - # if `user` is None, use `account` - *logins, password = auth_from_netrc - login = logins[0] if logins[0] else logins[-1] - auth = BasicAuth(cast(str, login), cast(str, password)) + try: + auth = basicauth_from_netrc(netrc_obj, proxy.host) + except LookupError: + auth = None ret[proto] = ProxyInfo(proxy, auth) return ret diff --git a/docs/client_reference.rst b/docs/client_reference.rst index cdf5badf798..9d88cb85a83 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -179,6 +179,11 @@ The client session supports the context manager protocol for self closing. Get proxy credentials from ``~/.netrc`` file if present. + Get HTTP Basic Auth credentials from :file:`~/.netrc` file if present. + + If :envvar:`NETRC` environment variable is set, read from file specified + there rather than from :file:`~/.netrc`. + .. seealso:: ``.netrc`` documentation: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html @@ -189,6 +194,10 @@ The client session supports the context manager protocol for self closing. Added support for ``~/.netrc`` file. + .. versionchanged:: 3.9 + + Added support for reading HTTP Basic Auth credentials from :file:`~/.netrc` file. + :param bool requote_redirect_url: Apply *URL requoting* for redirection URLs if automatic redirection is enabled (``True`` by default). diff --git a/docs/glossary.rst b/docs/glossary.rst index 497f901176b..81bfcfa654b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -89,6 +89,8 @@ It makes communication faster by getting rid of connection establishment for every request. + + nginx Nginx [engine x] is an HTTP and reverse proxy server, a mail @@ -153,3 +155,16 @@ A library for operating with URL objects. https://pypi.python.org/pypi/yarl + + +Environment Variables +===================== + +.. envvar:: NETRC + + If set, HTTP Basic Auth will be read from the file pointed to by this environment variable, + rather than from :file:`~/.netrc`. + + .. seealso:: + + ``.netrc`` documentation: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html diff --git a/tests/conftest.py b/tests/conftest.py index 6e0cf73f93c..36b55b20925 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,3 +197,25 @@ def selector_loop() -> None: with loop_context(policy.new_event_loop) as _loop: asyncio.set_event_loop(_loop) yield _loop + + +@pytest.fixture +def netrc_contents( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +): + """ + Prepare :file:`.netrc` with given contents. + + Monkey-patches :envvar:`NETRC` to point to created file. + """ + netrc_contents = getattr(request, "param", None) + + netrc_file_path = tmp_path / ".netrc" + if netrc_contents is not None: + netrc_file_path.write_text(netrc_contents) + + monkeypatch.setenv("NETRC", str(netrc_file_path)) + + return netrc_file_path diff --git a/tests/test_client_request.py b/tests/test_client_request.py index ecda7b0dc93..b0b11fda7e3 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -5,7 +5,7 @@ import pathlib import zlib from http.cookies import BaseCookie, Morsel, SimpleCookie -from typing import Any +from typing import Any, Optional from unittest import mock import pytest @@ -13,7 +13,7 @@ from yarl import URL import aiohttp -from aiohttp import BaseConnector, hdrs, payload +from aiohttp import BaseConnector, hdrs, helpers, payload from aiohttp.client_reqrep import ( ClientRequest, ClientResponse, @@ -1230,3 +1230,51 @@ def test_loose_cookies_types(loop: Any) -> None: def test_gen_default_accept_encoding(has_brotli: Any, expected: Any) -> None: with mock.patch("aiohttp.client_reqrep.HAS_BROTLI", has_brotli): assert _gen_default_accept_encoding() == expected + + +@pytest.mark.parametrize( + ("netrc_contents", "expected_auth"), + [ + ( + "machine example.com login username password pass\n", + helpers.BasicAuth("username", "pass"), + ) + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_netrc_present( + make_request: Any, + expected_auth: Optional[helpers.BasicAuth], +): + """Test appropriate Authorization header is sent when netrc is not empty.""" + req = make_request("get", "http://example.com", trust_env=True) + assert req.headers[hdrs.AUTHORIZATION] == expected_auth.encode() + + +@pytest.mark.parametrize( + "netrc_contents", + ("machine example.com login username password pass\n",), + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_netrc_present_untrusted_env( + make_request: Any, +): + """Test no authorization header is sent via netrc if trust_env is False""" + req = make_request("get", "http://example.com", trust_env=False) + assert hdrs.AUTHORIZATION not in req.headers + + +@pytest.mark.parametrize( + "netrc_contents", + ("",), + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_empty_netrc( + make_request: Any, +): + """Test that no Authorization header is sent when netrc is empty""" + req = make_request("get", "http://example.com", trust_env=True) + assert hdrs.AUTHORIZATION not in req.headers diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 77c00c4be14..aa1dcf6b214 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -974,3 +974,66 @@ def test_populate_with_cookies(): ) def test_parse_http_date(value, expected): assert parse_http_date(value) == expected + + +@pytest.mark.parametrize( + ["netrc_contents", "expected_username"], + [ + ( + "machine example.com login username password pass\n", + "username", + ), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_netrc_from_env(expected_username: str): + """Test that reading netrc files from env works as expected""" + netrc_obj = helpers.netrc_from_env() + assert netrc_obj.authenticators("example.com")[0] == expected_username + + +@pytest.mark.parametrize( + ["netrc_contents", "expected_auth"], + [ + ( + "machine example.com login username password pass\n", + helpers.BasicAuth("username", "pass"), + ), + ( + "machine example.com account username password pass\n", + helpers.BasicAuth("username", "pass"), + ), + ( + "machine example.com password pass\n", + helpers.BasicAuth("", "pass"), + ), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_present_in_netrc( + expected_auth: helpers.BasicAuth, +): + """Test that netrc file contents are properly parsed into BasicAuth tuples""" + netrc_obj = helpers.netrc_from_env() + + assert expected_auth == helpers.basicauth_from_netrc(netrc_obj, "example.com") + + +@pytest.mark.parametrize( + ["netrc_contents"], + [ + ("",), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_read_basicauth_from_empty_netrc(): + """Test that an error is raised if netrc doesn't have an entry for our host""" + netrc_obj = helpers.netrc_from_env() + + with pytest.raises( + LookupError, match="No entry for example.com found in the `.netrc` file." + ): + helpers.basicauth_from_netrc(netrc_obj, "example.com")