diff --git a/CHANGES/7131.feature b/CHANGES/7131.feature new file mode 100644 index 00000000000..bd77aff3613 --- /dev/null +++ b/CHANGES/7131.feature @@ -0,0 +1 @@ +Added support for using Basic Auth credentials from :file:`.netrc` file when making HTTP requests with the :py:class:`~aiohttp.ClientSession` ``trust_env`` argument is set to ``True`` -- by :user:`yuvipanda`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a934dee9967..28359ea100f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -355,6 +355,7 @@ Yury Pliner Yury Selivanov Yusuke Tsutsumi Yuval Ofir +Yuvi Panda Zainab Lawal Zeal Wierslee Zlatan Sičanica diff --git a/aiohttp/client.py b/aiohttp/client.py index 565cc0ea561..e1befbeb68e 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -481,6 +481,7 @@ async def _request( ssl=ssl, proxy_headers=proxy_headers, traces=traces, + trust_env=self.trust_env, ) # connection timeout diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index dadc4e2fa7b..edaf7087fc5 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -1,5 +1,6 @@ import asyncio import codecs +import contextlib import dataclasses import functools import io @@ -45,7 +46,9 @@ BasicAuth, HeadersMixin, TimerNoop, + basicauth_from_netrc, is_expected_content_type, + netrc_from_env, noop, parse_mimetype, reify, @@ -210,6 +213,7 @@ def __init__( ssl: Union[SSLContext, bool, Fingerprint, None] = None, proxy_headers: Optional[LooseHeaders] = None, traces: Optional[List["Trace"]] = None, + trust_env: bool = False, ): match = _CONTAINS_CONTROL_CHAR_RE.search(method) if match: @@ -251,7 +255,7 @@ def __init__( self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) - self.update_auth(auth) + self.update_auth(auth, trust_env) self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) @@ -428,10 +432,14 @@ def update_transfer_encoding(self) -> None: if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) - def update_auth(self, auth: Optional[BasicAuth]) -> None: + def update_auth(self, auth: Optional[BasicAuth], trust_env: bool = False) -> None: """Set basic auth.""" if auth is None: auth = self.auth + if auth is None and trust_env and self.url.host is not None: + netrc_obj = netrc_from_env() + with contextlib.suppress(LookupError): + auth = basicauth_from_netrc(netrc_obj, self.url.host) if auth is None: return diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index f3a9026393e..431bd67dd14 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -41,7 +41,6 @@ Type, TypeVar, Union, - cast, overload, ) from urllib.parse import quote @@ -244,6 +243,35 @@ class ProxyInfo: proxy_auth: Optional[BasicAuth] +def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth: + """ + Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``. + + :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no + entry is found for the ``host``. + """ + if netrc_obj is None: + raise LookupError("No .netrc file found") + auth_from_netrc = netrc_obj.authenticators(host) + + if auth_from_netrc is None: + raise LookupError(f"No entry for {host!s} found in the `.netrc` file.") + login, account, password = auth_from_netrc + + # TODO(PY311): username = login or account + # Up to python 3.10, account could be None if not specified, + # and login will be empty string if not specified. From 3.11, + # login and account will be empty string if not specified. + username = login if (login or account is None) else account + + # TODO(PY311): Remove this, as password will be empty string + # if not specified + if password is None: + password = "" + + return BasicAuth(username, password) + + def proxies_from_env() -> Dict[str, ProxyInfo]: proxy_urls = { k: URL(v) @@ -261,16 +289,11 @@ def proxies_from_env() -> Dict[str, ProxyInfo]: ) continue if netrc_obj and auth is None: - auth_from_netrc = None if proxy.host is not None: - auth_from_netrc = netrc_obj.authenticators(proxy.host) - if auth_from_netrc is not None: - # auth_from_netrc is a (`user`, `account`, `password`) tuple, - # `user` and `account` both can be username, - # if `user` is None, use `account` - *logins, password = auth_from_netrc - login = logins[0] if logins[0] else logins[-1] - auth = BasicAuth(cast(str, login), cast(str, password)) + try: + auth = basicauth_from_netrc(netrc_obj, proxy.host) + except LookupError: + auth = None ret[proto] = ProxyInfo(proxy, auth) return ret diff --git a/docs/client_reference.rst b/docs/client_reference.rst index cdf5badf798..9d88cb85a83 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -179,6 +179,11 @@ The client session supports the context manager protocol for self closing. Get proxy credentials from ``~/.netrc`` file if present. + Get HTTP Basic Auth credentials from :file:`~/.netrc` file if present. + + If :envvar:`NETRC` environment variable is set, read from file specified + there rather than from :file:`~/.netrc`. + .. seealso:: ``.netrc`` documentation: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html @@ -189,6 +194,10 @@ The client session supports the context manager protocol for self closing. Added support for ``~/.netrc`` file. + .. versionchanged:: 3.9 + + Added support for reading HTTP Basic Auth credentials from :file:`~/.netrc` file. + :param bool requote_redirect_url: Apply *URL requoting* for redirection URLs if automatic redirection is enabled (``True`` by default). diff --git a/docs/glossary.rst b/docs/glossary.rst index 497f901176b..81bfcfa654b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -89,6 +89,8 @@ It makes communication faster by getting rid of connection establishment for every request. + + nginx Nginx [engine x] is an HTTP and reverse proxy server, a mail @@ -153,3 +155,16 @@ A library for operating with URL objects. https://pypi.python.org/pypi/yarl + + +Environment Variables +===================== + +.. envvar:: NETRC + + If set, HTTP Basic Auth will be read from the file pointed to by this environment variable, + rather than from :file:`~/.netrc`. + + .. seealso:: + + ``.netrc`` documentation: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html diff --git a/tests/conftest.py b/tests/conftest.py index 6e0cf73f93c..36b55b20925 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,3 +197,25 @@ def selector_loop() -> None: with loop_context(policy.new_event_loop) as _loop: asyncio.set_event_loop(_loop) yield _loop + + +@pytest.fixture +def netrc_contents( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +): + """ + Prepare :file:`.netrc` with given contents. + + Monkey-patches :envvar:`NETRC` to point to created file. + """ + netrc_contents = getattr(request, "param", None) + + netrc_file_path = tmp_path / ".netrc" + if netrc_contents is not None: + netrc_file_path.write_text(netrc_contents) + + monkeypatch.setenv("NETRC", str(netrc_file_path)) + + return netrc_file_path diff --git a/tests/test_client_request.py b/tests/test_client_request.py index ecda7b0dc93..b0b11fda7e3 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -5,7 +5,7 @@ import pathlib import zlib from http.cookies import BaseCookie, Morsel, SimpleCookie -from typing import Any +from typing import Any, Optional from unittest import mock import pytest @@ -13,7 +13,7 @@ from yarl import URL import aiohttp -from aiohttp import BaseConnector, hdrs, payload +from aiohttp import BaseConnector, hdrs, helpers, payload from aiohttp.client_reqrep import ( ClientRequest, ClientResponse, @@ -1230,3 +1230,51 @@ def test_loose_cookies_types(loop: Any) -> None: def test_gen_default_accept_encoding(has_brotli: Any, expected: Any) -> None: with mock.patch("aiohttp.client_reqrep.HAS_BROTLI", has_brotli): assert _gen_default_accept_encoding() == expected + + +@pytest.mark.parametrize( + ("netrc_contents", "expected_auth"), + [ + ( + "machine example.com login username password pass\n", + helpers.BasicAuth("username", "pass"), + ) + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_netrc_present( + make_request: Any, + expected_auth: Optional[helpers.BasicAuth], +): + """Test appropriate Authorization header is sent when netrc is not empty.""" + req = make_request("get", "http://example.com", trust_env=True) + assert req.headers[hdrs.AUTHORIZATION] == expected_auth.encode() + + +@pytest.mark.parametrize( + "netrc_contents", + ("machine example.com login username password pass\n",), + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_netrc_present_untrusted_env( + make_request: Any, +): + """Test no authorization header is sent via netrc if trust_env is False""" + req = make_request("get", "http://example.com", trust_env=False) + assert hdrs.AUTHORIZATION not in req.headers + + +@pytest.mark.parametrize( + "netrc_contents", + ("",), + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_from_empty_netrc( + make_request: Any, +): + """Test that no Authorization header is sent when netrc is empty""" + req = make_request("get", "http://example.com", trust_env=True) + assert hdrs.AUTHORIZATION not in req.headers diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 77c00c4be14..aa1dcf6b214 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -974,3 +974,66 @@ def test_populate_with_cookies(): ) def test_parse_http_date(value, expected): assert parse_http_date(value) == expected + + +@pytest.mark.parametrize( + ["netrc_contents", "expected_username"], + [ + ( + "machine example.com login username password pass\n", + "username", + ), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_netrc_from_env(expected_username: str): + """Test that reading netrc files from env works as expected""" + netrc_obj = helpers.netrc_from_env() + assert netrc_obj.authenticators("example.com")[0] == expected_username + + +@pytest.mark.parametrize( + ["netrc_contents", "expected_auth"], + [ + ( + "machine example.com login username password pass\n", + helpers.BasicAuth("username", "pass"), + ), + ( + "machine example.com account username password pass\n", + helpers.BasicAuth("username", "pass"), + ), + ( + "machine example.com password pass\n", + helpers.BasicAuth("", "pass"), + ), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_basicauth_present_in_netrc( + expected_auth: helpers.BasicAuth, +): + """Test that netrc file contents are properly parsed into BasicAuth tuples""" + netrc_obj = helpers.netrc_from_env() + + assert expected_auth == helpers.basicauth_from_netrc(netrc_obj, "example.com") + + +@pytest.mark.parametrize( + ["netrc_contents"], + [ + ("",), + ], + indirect=("netrc_contents",), +) +@pytest.mark.usefixtures("netrc_contents") +def test_read_basicauth_from_empty_netrc(): + """Test that an error is raised if netrc doesn't have an entry for our host""" + netrc_obj = helpers.netrc_from_env() + + with pytest.raises( + LookupError, match="No entry for example.com found in the `.netrc` file." + ): + helpers.basicauth_from_netrc(netrc_obj, "example.com")