diff --git a/CHANGES.rst b/CHANGES.rst index ca5c7ff9..6ae2a2a6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ Changes ------- +2.11.2 (2024-01-27) +^^^^^^^^^^^^^^^^^^^ +* fix use of proxies #1070 + 2.11.1 (2024-01-25) ^^^^^^^^^^^^^^^^^^^ * bump botocore dependency specification diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index d3583e9b..ded7bd35 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1 +1 @@ -__version__ = '2.11.1' +__version__ = '2.11.2' diff --git a/aiobotocore/httpsession.py b/aiobotocore/httpsession.py index b98c59ed..f8224899 100644 --- a/aiobotocore/httpsession.py +++ b/aiobotocore/httpsession.py @@ -55,7 +55,8 @@ def __init__( connector_args=None, ): # TODO: handle socket_options - self._session: Optional[aiohttp.ClientSession] = None + # keep track of sessions by proxy url (if any) + self._sessions: Dict[Optional[str], aiohttp.ClientSession] = {} self._verify = verify self._proxy_config = ProxyConfiguration( proxies=proxies, proxies_settings=proxies_config @@ -93,53 +94,23 @@ def __init__( # it also pools by host so we don't need a manager, and can pass proxy via # request so don't need proxy manager - ssl_context = None - if bool(verify): - if proxies: - proxies_settings = self._proxy_config.settings - ssl_context = self._setup_proxy_ssl_context(proxies_settings) - # TODO: add support for - # proxies_settings.get('proxy_use_forwarding_for_https') - else: - ssl_context = self._get_ssl_context() - - # inline self._setup_ssl_cert - ca_certs = get_cert_path(verify) - if ca_certs: - ssl_context.load_verify_locations(ca_certs, None, None) - - self._create_connector = lambda: aiohttp.TCPConnector( - limit=max_pool_connections, - verify_ssl=bool(verify), - ssl=ssl_context, - **self._connector_args - ) - self._connector = None - async def __aenter__(self): - assert not self._session and not self._connector - - self._connector = self._create_connector() + assert not self._sessions - self._session = aiohttp.ClientSession( - connector=self._connector, - timeout=self._timeout, - skip_auto_headers={'CONTENT-TYPE'}, - auto_decompress=False, - ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): - if self._session: - await self._session.__aexit__(exc_type, exc_val, exc_tb) - self._session = None - self._connector = None + if sessions := tuple(self._sessions.values()): + self._sessions.clear() + await asyncio.gather( + *( + session.__aexit__(exc_type, exc_val, exc_tb) + for session in sessions + ), + ) def _get_ssl_context(self): - ssl_context = create_urllib3_context() - if self._cert_file: - ssl_context.load_cert_chain(self._cert_file, self._key_file) - return ssl_context + return create_urllib3_context() def _setup_proxy_ssl_context(self, proxy_url): proxies_settings = self._proxy_config.settings @@ -167,6 +138,35 @@ def _setup_proxy_ssl_context(self, proxy_url): except (OSError, LocationParseError) as e: raise InvalidProxiesConfigError(error=e) + def _create_connector(self, proxy_url): + ssl_context = None + if bool(self._verify): + if proxy_url: + ssl_context = self._setup_proxy_ssl_context(proxy_url) + # TODO: add support for + # proxies_settings.get('proxy_use_forwarding_for_https') + else: + ssl_context = self._get_ssl_context() + + if ssl_context: + if self._cert_file: + ssl_context.load_cert_chain( + self._cert_file, + self._key_file, + ) + + # inline self._setup_ssl_cert + ca_certs = get_cert_path(self._verify) + if ca_certs: + ssl_context.load_verify_locations(ca_certs, None, None) + + return aiohttp.TCPConnector( + limit=self._max_pool_connections, + verify_ssl=bool(self._verify), + ssl=ssl_context, + **self._connector_args, + ) + async def close(self): await self.__aexit__(None, None, None) @@ -205,7 +205,18 @@ async def send(self, request): data = _IOBaseWrapper(data) url = URL(url, encoded=True) - response = await self._session.request( + + session = self._sessions.get(proxy_url) + if session is None: + connector = self._create_connector(proxy_url) + self._sessions[proxy_url] = session = aiohttp.ClientSession( + connector=connector, + timeout=self._timeout, + skip_auto_headers={'CONTENT-TYPE'}, + auto_decompress=False, + ) + + response = await session.request( request.method, url=url, chunked=chunked,