From 3604e232b45696392acabafee834ec130ddc2d85 Mon Sep 17 00:00:00 2001 From: Victor K Date: Mon, 30 Oct 2017 14:54:32 +0200 Subject: [PATCH 01/33] Fix wrap oserrors. (#2442) * Fix wrap oserrors. * Skip unix socket tests on windows --- CHANGES/2423.bugfix | 1 + aiohttp/connector.py | 15 ++++++++++++-- tests/test_connector.py | 44 ++++++++++++++++++++++++++++++++++++++++- tests/test_proxy.py | 2 +- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 CHANGES/2423.bugfix diff --git a/CHANGES/2423.bugfix b/CHANGES/2423.bugfix new file mode 100644 index 00000000000..08823f4f69f --- /dev/null +++ b/CHANGES/2423.bugfix @@ -0,0 +1 @@ +Fix connector convert OSError to ClientConnectorError diff --git a/aiohttp/connector.py b/aiohttp/connector.py index e70072148aa..47f8f3fb95b 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -785,6 +785,13 @@ def _create_direct_connection(self, req): sslcontext = self._get_ssl_context(req) fingerprint, hashfunc = self._get_fingerprint_and_hashfunc(req) + try: + hosts = yield from self._resolve_host(req.url.raw_host, req.port) + except OSError as exc: + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorError(req.connection_key, exc) from exc + hosts = yield from self._resolve_host(req.url.raw_host, req.port) for hinfo in hosts: @@ -938,6 +945,10 @@ def path(self): @asyncio.coroutine def _create_connection(self, req): - _, proto = yield from self._loop.create_unix_connection( - self._factory, self._path) + try: + _, proto = yield from self._loop.create_unix_connection( + self._factory, self._path) + except OSError as exc: + raise ClientConnectorError(req.connection_key, exc) from exc + return proto diff --git a/tests/test_connector.py b/tests/test_connector.py index 68167174e68..2970a9aba8e 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -9,6 +9,7 @@ import ssl import tempfile import unittest +import uuid from unittest import mock import pytest @@ -18,7 +19,7 @@ from aiohttp import client, helpers, web from aiohttp.client import ClientRequest from aiohttp.connector import Connection, _DNSCacheTable -from aiohttp.test_utils import unused_port +from aiohttp.test_utils import make_mocked_coro, unused_port @pytest.fixture() @@ -507,6 +508,19 @@ def test_tcp_connector_dns_throttle_requests_cancelled_when_close( yield from f +def test_dns_error(loop): + connector = aiohttp.TCPConnector(loop=loop) + connector._resolve_host = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_get_pop_empty_conns(loop): # see issue #473 conn = aiohttp.BaseConnector(loop=loop) @@ -1281,6 +1295,34 @@ def handler(request): assert r.status == 200 +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_not_found(loop): + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_permission(loop): + loop.create_unix_connection = make_mocked_coro( + raise_exception=PermissionError()) + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_default_use_dns_cache(loop): conn = aiohttp.TCPConnector(loop=loop) assert conn.use_dns_cache diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 9a0380f520f..ab123c05a56 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -215,7 +215,7 @@ def _test_connect_request_with_unicode_host(self, Request_mock): Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80", mock.ANY, loop=loop) - def test_proxy_connection_error(self): + def test_proxy_dns_error(self): connector = aiohttp.TCPConnector(loop=self.loop) connector._resolve_host = make_mocked_coro( raise_exception=OSError('dont take it serious')) From 7f31c58f9bff40b95e0e58455ae92e7eb57a3074 Mon Sep 17 00:00:00 2001 From: Victor K Date: Mon, 30 Oct 2017 17:12:12 +0200 Subject: [PATCH 02/33] Fix wrap ssl errors for proxy connector. (#2446) --- CHANGES/2408.bugfix | 1 + aiohttp/connector.py | 99 +++++++++++++++++---------------- tests/test_proxy.py | 129 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 158 insertions(+), 71 deletions(-) create mode 100644 CHANGES/2408.bugfix diff --git a/CHANGES/2408.bugfix b/CHANGES/2408.bugfix new file mode 100644 index 00000000000..4fec0c8b07c --- /dev/null +++ b/CHANGES/2408.bugfix @@ -0,0 +1 @@ +Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy connector diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 47f8f3fb95b..fbad64c4435 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -781,7 +781,22 @@ def _get_fingerprint_and_hashfunc(self, req): return (None, None) @asyncio.coroutine - def _create_direct_connection(self, req): + def _wrap_create_connection(self, *args, + req, client_error=ClientConnectorError, + **kwargs): + try: + return (yield from self._loop.create_connection(*args, **kwargs)) + except certificate_errors as exc: + raise ClientConnectorCertificateError( + req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + raise client_error(req.connection_key, exc) from exc + + @asyncio.coroutine + def _create_direct_connection(self, req, + *, client_error=ClientConnectorError): sslcontext = self._get_ssl_context(req) fingerprint, hashfunc = self._get_fingerprint_and_hashfunc(req) @@ -792,45 +807,36 @@ def _create_direct_connection(self, req): # it is problem of resolving proxy ip itself raise ClientConnectorError(req.connection_key, exc) from exc - hosts = yield from self._resolve_host(req.url.raw_host, req.port) - for hinfo in hosts: - try: - host = hinfo['host'] - port = hinfo['port'] - transp, proto = yield from self._loop.create_connection( - self._factory, host, port, - ssl=sslcontext, family=hinfo['family'], - proto=hinfo['proto'], flags=hinfo['flags'], - server_hostname=hinfo['hostname'] if sslcontext else None, - local_addr=self._local_addr) - has_cert = transp.get_extra_info('sslcontext') - if has_cert and fingerprint: - sock = transp.get_extra_info('socket') - if not hasattr(sock, 'getpeercert'): - # Workaround for asyncio 3.5.0 - # Starting from 3.5.1 version - # there is 'ssl_object' extra info in transport - sock = transp._ssl_protocol._sslpipe.ssl_object - # gives DER-encoded cert as a sequence of bytes (or None) - cert = sock.getpeercert(binary_form=True) - assert cert - got = hashfunc(cert).digest() - expected = fingerprint - if got != expected: - transp.close() - if not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transp) - raise ServerFingerprintMismatch( - expected, got, host, port) - return transp, proto - except certificate_errors as exc: - raise ClientConnectorCertificateError( - req.connection_key, exc) from exc - except ssl_errors as exc: - raise ClientConnectorSSLError(req.connection_key, exc) from exc - except OSError as exc: - raise ClientConnectorError(req.connection_key, exc) from exc + host = hinfo['host'] + port = hinfo['port'] + transp, proto = yield from self._wrap_create_connection( + self._factory, host, port, + ssl=sslcontext, family=hinfo['family'], + proto=hinfo['proto'], flags=hinfo['flags'], + server_hostname=hinfo['hostname'] if sslcontext else None, + local_addr=self._local_addr, + req=req, client_error=client_error) + has_cert = transp.get_extra_info('sslcontext') + if has_cert and fingerprint: + sock = transp.get_extra_info('socket') + if not hasattr(sock, 'getpeercert'): + # Workaround for asyncio 3.5.0 + # Starting from 3.5.1 version + # there is 'ssl_object' extra info in transport + sock = transp._ssl_protocol._sslpipe.ssl_object + # gives DER-encoded cert as a sequence of bytes (or None) + cert = sock.getpeercert(binary_form=True) + assert cert + got = hashfunc(cert).digest() + expected = fingerprint + if got != expected: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + raise ServerFingerprintMismatch( + expected, got, host, port) + return transp, proto @asyncio.coroutine def _create_proxy_connection(self, req): @@ -847,12 +853,10 @@ def _create_proxy_connection(self, req): verify_ssl=req.verify_ssl, fingerprint=req.fingerprint, ssl_context=req.ssl_context) - try: - # create connection to proxy server - transport, proto = yield from self._create_direct_connection( - proxy_req) - except OSError as exc: - raise ClientProxyConnectionError(proxy_req, exc) from exc + + # create connection to proxy server + transport, proto = yield from self._create_direct_connection( + proxy_req, client_error=ClientProxyConnectionError) auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) if auth is not None: @@ -903,9 +907,10 @@ def _create_proxy_connection(self, req): finally: transport.close() - transport, proto = yield from self._loop.create_connection( + transport, proto = yield from self._wrap_create_connection( self._factory, ssl=sslcontext, sock=rawsock, - server_hostname=req.host) + server_hostname=req.host, + req=req) finally: proxy_resp.close() diff --git a/tests/test_proxy.py b/tests/test_proxy.py index ab123c05a56..cc066904e2b 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -2,6 +2,7 @@ import gc import hashlib import socket +import ssl import unittest from unittest import mock @@ -231,6 +232,24 @@ def test_proxy_dns_error(self): self.assertEqual(req.url.path, '/') self.assertEqual(dict(req.headers), expected_headers) + def test_proxy_connection_error(self): + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro([{ + 'hostname': 'www.python.org', + 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, + 'flags': socket.AI_NUMERICHOST}]) + connector._loop.create_connection = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientProxyConnectionError): + self.loop.run_until_complete(connector.connect(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_auth(self, ClientRequestMock): proxy_req = ClientRequest( @@ -314,30 +333,6 @@ def test_auth_from_url(self, ClientRequestMock): ssl_context=None, verify_ssl=None) conn.close() - @mock.patch('aiohttp.connector.ClientRequest') - def test_auth__not_modifying_request(self, ClientRequestMock): - proxy_req = ClientRequest('GET', - URL('http://user:pass@proxy.example.com'), - loop=self.loop) - ClientRequestMock.return_value = proxy_req - proxy_req_headers = dict(proxy_req.headers) - - connector = aiohttp.TCPConnector(loop=self.loop) - connector._resolve_host = make_mocked_coro( - raise_exception=OSError('nothing personal')) - - req = ClientRequest( - 'GET', URL('http://www.python.org'), - proxy=URL('http://user:pass@proxy.example.com'), - loop=self.loop, - ) - req_headers = dict(req.headers) - with self.assertRaises(aiohttp.ClientConnectorError): - self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.headers, req_headers) - self.assertEqual(req.url.path, '/') - self.assertEqual(proxy_req.headers, proxy_req_headers) - @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), @@ -375,6 +370,92 @@ def test_https_connect(self, ClientRequestMock): proxy_resp.close() self.loop.run_until_complete(req.close()) + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_certificate_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.CertificateError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorCertificateError): + self.loop.run_until_complete(connector._create_connection(req)) + + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_ssl_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.SSLError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorSSLError): + self.loop.run_until_complete(connector._create_connection(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_runtime_error(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), From 3c67ae363a3b5ee16a533396a8761fc72e07493b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 31 Oct 2017 11:36:18 +0200 Subject: [PATCH 03/33] Fix ABC signature --- aiohttp/abc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/abc.py b/aiohttp/abc.py index 8cee25bc8e4..70513be6f0a 100644 --- a/aiohttp/abc.py +++ b/aiohttp/abc.py @@ -84,7 +84,7 @@ def request(self): return self._request @abstractmethod - async def __await__(self): + def __await__(self): """Execute the view handler.""" From fab1f237ec538228c0ef47546dc0278d6cd967d2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 31 Oct 2017 15:00:49 +0200 Subject: [PATCH 04/33] Fix slot name --- aiohttp/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 8c1824fdf91..2e0f460e35b 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -43,7 +43,7 @@ class _BaseCoroMixin(Coroutine): - __slots__ = ('_coro') + __slots__ = ('_coro',) def __init__(self, coro): self._coro = coro From 6e53ed4886a82b445cea3bea08337f5ef927970a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 31 Oct 2017 15:27:16 +0200 Subject: [PATCH 05/33] Fix 2451: Rename from_env to trust_env in client reference. --- CHANGES/2451.doc | 1 + docs/client_reference.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 CHANGES/2451.doc diff --git a/CHANGES/2451.doc b/CHANGES/2451.doc new file mode 100644 index 00000000000..386917d1926 --- /dev/null +++ b/CHANGES/2451.doc @@ -0,0 +1 @@ +Rename `from_env` to `trust_env` in client reference. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 101aa3163d7..8f54b6391c2 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -143,7 +143,7 @@ The client session supports the context manager protocol for self closing. .. versionadded:: 2.3 - :param bool from_env: Get proxies information from *HTTP_PROXY* / + :param bool trust_env: Get proxies information from *HTTP_PROXY* / *HTTPS_PROXY* environment variables if the parameter is ``True`` (``False`` by default). From 4f7c91bfdc71d0de7e22b146d20c1adc718b8004 Mon Sep 17 00:00:00 2001 From: Amin Etesamian Date: Tue, 31 Oct 2017 20:37:48 +0330 Subject: [PATCH 06/33] payload.py now uses async/await syntax (#2452) --- aiohttp/payload.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index 7296594c90b..fca80e7dc2b 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -126,9 +126,8 @@ def set_content_disposition(self, disptype, quote_fields=True, **params): self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header( disptype, quote_fields=quote_fields, **params) - @asyncio.coroutine # pragma: no branch @abstractmethod - def write(self, writer): + async def write(self, writer): """Write payload. writer is an AbstractPayloadWriter instance: @@ -234,12 +233,11 @@ def size(self): except OSError: return None - @asyncio.coroutine - def write(self, writer): + async def write(self, writer): try: chunk = self._value.read(DEFAULT_LIMIT) while chunk: - yield from writer.write(chunk.encode(self._encoding)) + await writer.write(chunk.encode(self._encoding)) chunk = self._value.read(DEFAULT_LIMIT) finally: self._value.close() @@ -269,24 +267,22 @@ def size(self): class StreamReaderPayload(Payload): - @asyncio.coroutine - def write(self, writer): - chunk = yield from self._value.read(DEFAULT_LIMIT) + async def write(self, writer): + chunk = await self._value.read(DEFAULT_LIMIT) while chunk: - yield from writer.write(chunk) - chunk = yield from self._value.read(DEFAULT_LIMIT) + await writer.write(chunk) + chunk = await self._value.read(DEFAULT_LIMIT) class DataQueuePayload(Payload): - @asyncio.coroutine - def write(self, writer): + async def write(self, writer): while True: try: - chunk = yield from self._value.read() + chunk = await self._value.read() if not chunk: break - yield from writer.write(chunk) + await writer.write(chunk) except EofStream: break From 460e71948cf2c87ea0041198b38e7f3f65200263 Mon Sep 17 00:00:00 2001 From: Victor K Date: Wed, 1 Nov 2017 11:29:44 +0200 Subject: [PATCH 07/33] Fix connection for multiple dns hosts (#2447) * Proof of concept fix for multiple dns hosts * Revert getpeercert * Added test for multiple dns hosts and errors. * Added change notes. * Remove redunant assert. --- CHANGES/2424.bugfix | 1 + aiohttp/connector.py | 28 ++++++++--- tests/test_connector.py | 106 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 CHANGES/2424.bugfix diff --git a/CHANGES/2424.bugfix b/CHANGES/2424.bugfix new file mode 100644 index 00000000000..f06477b975d --- /dev/null +++ b/CHANGES/2424.bugfix @@ -0,0 +1 @@ +Fix connection attempts for multiple dns hosts diff --git a/aiohttp/connector.py b/aiohttp/connector.py index fbad64c4435..c67ad78bf92 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -807,16 +807,24 @@ def _create_direct_connection(self, req, # it is problem of resolving proxy ip itself raise ClientConnectorError(req.connection_key, exc) from exc + last_exc = None + for hinfo in hosts: host = hinfo['host'] port = hinfo['port'] - transp, proto = yield from self._wrap_create_connection( - self._factory, host, port, - ssl=sslcontext, family=hinfo['family'], - proto=hinfo['proto'], flags=hinfo['flags'], - server_hostname=hinfo['hostname'] if sslcontext else None, - local_addr=self._local_addr, - req=req, client_error=client_error) + + try: + transp, proto = yield from self._wrap_create_connection( + self._factory, host, port, + ssl=sslcontext, family=hinfo['family'], + proto=hinfo['proto'], flags=hinfo['flags'], + server_hostname=hinfo['hostname'] if sslcontext else None, + local_addr=self._local_addr, + req=req, client_error=client_error) + except ClientConnectorError as exc: + last_exc = exc + continue + has_cert = transp.get_extra_info('sslcontext') if has_cert and fingerprint: sock = transp.get_extra_info('socket') @@ -834,9 +842,13 @@ def _create_direct_connection(self, req, transp.close() if not self._cleanup_closed_disabled: self._cleanup_closed_transports.append(transp) - raise ServerFingerprintMismatch( + last_exc = ServerFingerprintMismatch( expected, got, host, port) + continue + return transp, proto + else: + raise last_exc @asyncio.coroutine def _create_proxy_connection(self, req): diff --git a/tests/test_connector.py b/tests/test_connector.py index 2970a9aba8e..ddcf67c574b 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -2,6 +2,7 @@ import asyncio import gc +import hashlib import os.path import platform import shutil @@ -363,6 +364,111 @@ def certificate_error(*args, **kwargs): '[CertificateError: ()]') +@asyncio.coroutine +def test_tcp_connector_multiple_hosts_errors(loop): + conn = aiohttp.TCPConnector(loop=loop) + + ip1 = '192.168.1.1' + ip2 = '192.168.1.2' + ip3 = '192.168.1.3' + ip4 = '192.168.1.4' + ip5 = '192.168.1.5' + ips = [ip1, ip2, ip3, ip4, ip5] + ips_tried = [] + + fingerprint = hashlib.sha256(b'foo').digest() + + req = ClientRequest('GET', URL('https://mocked.host'), + fingerprint=fingerprint, + loop=loop) + + @asyncio.coroutine + def _resolve_host(host, port): + return [{ + 'hostname': host, + 'host': ip, + 'port': port, + 'family': socket.AF_INET, + 'proto': 0, + 'flags': socket.AI_NUMERICHOST} + for ip in ips] + + conn._resolve_host = _resolve_host + + os_error = certificate_error = ssl_error = fingerprint_error = False + connected = False + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal os_error, certificate_error, ssl_error, fingerprint_error + nonlocal connected + + ip = args[1] + + ips_tried.append(ip) + + if ip == ip1: + os_error = True + raise OSError + + if ip == ip2: + certificate_error = True + raise ssl.CertificateError + + if ip == ip3: + ssl_error = True + raise ssl.SSLError + + if ip == ip4: + fingerprint_error = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'not foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + if ip == ip5: + connected = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + assert False + + conn._loop.create_connection = create_connection + + yield from conn.connect(req) + assert ips == ips_tried + + assert os_error + assert certificate_error + assert ssl_error + assert fingerprint_error + assert connected + + @asyncio.coroutine def test_tcp_connector_resolve_host(loop): conn = aiohttp.TCPConnector(loop=loop, use_dns_cache=True) From 05f1562ad0c1c6a5e69f0edd6023072ca0f1cae9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 11:29:07 +0200 Subject: [PATCH 08/33] Minor docs formatting --- CHANGES.rst | 6 +++--- docs/deployment.rst | 6 +++--- docs/index.rst | 29 +++++++++++++++-------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 17ceb3f5280..b45122c6af7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,6 @@ -======= -Changes -======= +========= +Changelog +========= .. You should *NOT* be adding new change log entries to this file, this diff --git a/docs/deployment.rst b/docs/deployment.rst index 0acab53ef13..738a488a00c 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,8 +1,8 @@ .. _aiohttp-deployment: -========================= -aiohttp server deployment -========================= +================= +Server Deployment +================= There are several options for aiohttp server deployment: diff --git a/docs/index.rst b/docs/index.rst index 110a4ef38dd..24998aec4b8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,17 +3,18 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -aiohttp: Asynchronous HTTP Client/Server -======================================== +=============================================================== +aiohttp: Asynchronous HTTP Client/Server for Python and asyncio +=============================================================== -HTTP client/server for :term:`asyncio` (:pep:`3156`). +HTTP client/server for :term:`asyncio` and Python. .. _GitHub: https://github.com/aio-libs/aiohttp .. _Freenode: http://freenode.net Key Features ------------- +============ - Supports both :ref:`aiohttp-client` and :ref:`HTTP Server `. - Supports both :ref:`Server WebSockets ` and @@ -22,7 +23,7 @@ Key Features :ref:`aiohttp-web-signals` and pluggable routing. Library Installation --------------------- +==================== .. code-block:: bash @@ -44,7 +45,7 @@ This option is highly recommended: $ pip install aiodns Getting Started ---------------- +=============== Client example:: @@ -100,13 +101,13 @@ Server example:: Tutorial --------- +======== :ref:`Polls tutorial ` Source code ------------ +=========== The project is hosted on GitHub_ @@ -119,7 +120,7 @@ Continuous Integration. Dependencies ------------- +============ - Python 3.4.2+ - *chardet* @@ -144,7 +145,7 @@ Dependencies Communication channels ----------------------- +====================== *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs @@ -157,14 +158,14 @@ We support `Stack Overflow Please add *aiohttp* tag to your question there. Contributing ------------- +============ Please read the :ref:`instructions for contributors` before making a Pull Request. Authors and License -------------------- +=================== The ``aiohttp`` package is written mostly by Nikolay Kim and Andrew Svetlov. @@ -176,7 +177,7 @@ Feel free to improve this package and send a pull request to GitHub_. .. _aiohttp-backward-compatibility-policy: Policy for Backward Incompatible Changes ----------------------------------------- +======================================== *aiohttp* keeps backward compatibility. @@ -195,7 +196,7 @@ these changes as rare as possible. Contents --------- +======== .. toctree:: From a3c5818b0dd98e856fbcbba666107102e7ff81c5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 11:56:17 +0200 Subject: [PATCH 09/33] Move TOC to separate RST file --- docs/conf.py | 2 +- docs/index.rst | 33 +++------------------------------ docs/toc.rst | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 31 deletions(-) create mode 100644 docs/toc.rst diff --git a/docs/conf.py b/docs/conf.py index 67cbd10ea0f..f27810e46bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -86,7 +86,7 @@ # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = 'toc' # General information about the project. project = 'aiohttp' diff --git a/docs/index.rst b/docs/index.rst index 24998aec4b8..35a22dc25b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -195,34 +195,7 @@ solved without major API change, but we are working hard for keeping these changes as rare as possible. -Contents -======== +Table Of Contents +================= -.. toctree:: - - client - client_reference - tutorial - web - web_reference - web_lowlevel - abc - multipart - streams - api - logging - testing - deployment - faq - external - essays - contributing - changes - glossary - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +To see the full table of contents open the :ref:`link `. diff --git a/docs/toc.rst b/docs/toc.rst new file mode 100644 index 00000000000..b094ac42821 --- /dev/null +++ b/docs/toc.rst @@ -0,0 +1,35 @@ +Table of Contents +================= + +.. toctree:: + :caption: Table of Contents + :name: mastertoc + + Introduction + client + client_reference + tutorial + web + web_reference + web_lowlevel + abc + multipart + streams + api + logging + testing + deployment + faq + external + essays + contributing + changes + glossary + Sitemap + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From 2315183bab1ff22fa69b1d7a2fe322655c6346dc Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 12:00:56 +0200 Subject: [PATCH 10/33] Build tarball on appveyor --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 68c055a705d..22f6b1fb9ea 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -20,7 +20,7 @@ test_script: - "tools/build.cmd %PYTHON%\\python.exe setup.py test" after_test: - - "tools/build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + - "tools/build.cmd %PYTHON%\\python.exe setup.py sdist bdist_wheel" artifacts: - path: dist\* From c3edcd08cc0af46efc898244013cb083dab1ea45 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 12:01:17 +0200 Subject: [PATCH 11/33] Bump to 2.3.2b1 --- aiohttp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 21a8d4cb8e7..602f9cfdcd0 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.1' +__version__ = '2.3.2b1' # This relies on each of the submodules having an __all__ variable. From e9c266065e1e3d94f78c5711ad72cbed670cbdf5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 12:50:02 +0200 Subject: [PATCH 12/33] Update PYPI password for appveyor config --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 22f6b1fb9ea..5fcc9c5ca8e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,6 +1,6 @@ environment: PYPI_PASSWD: - secure: syNUF3e8AEPY327rWBkKag== + secure: u+K6dKi7+CXXVFEUG4V7zUyV3w7Ntg0Ork/RGVV0eSQ= matrix: - PYTHON: "C:\\Python34" - PYTHON: "C:\\Python34-x64" @@ -28,7 +28,7 @@ artifacts: deploy_script: - ps: >- if($env:appveyor_repo_tag -eq 'True') { - Invoke-Expression "$env:PYTHON\\python.exe -m twine upload dist/* --username fafhrd --password $env:PYPI_PASSWD" + Invoke-Expression "$env:PYTHON\\python.exe -m twine upload dist/* --username andrew.svetlov --password $env:PYPI_PASSWD" } #notifications: From 84863a8cf9f8df5595ddb189d680774776d16cfb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 12:50:19 +0200 Subject: [PATCH 13/33] Bump to 2.3.2b2 --- aiohttp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 602f9cfdcd0..6cb53a6eecb 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.2b1' +__version__ = '2.3.2b2' # This relies on each of the submodules having an __all__ variable. From 7dea13fe4787551a8461af39fbc2444b205cdc95 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 13:13:51 +0200 Subject: [PATCH 14/33] Bump to 2.3.2b3 --- .appveyor.yml | 2 +- aiohttp/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 5fcc9c5ca8e..b0294bbf823 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,7 +11,7 @@ environment: - PYTHON: "C:\\Python36-x64" install: - - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" + - "tools/build.cmd %PYTHON%\\python.exe -m pip install -U wheel setuptools" - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt" build: false diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 6cb53a6eecb..9d8552410df 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.2b2' +__version__ = '2.3.2b3' # This relies on each of the submodules having an __all__ variable. From 788c328376300c6e479576991c273ace830e82a6 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 13:37:14 +0200 Subject: [PATCH 15/33] Relax tie check to always pass on CI bots --- tests/test_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_connector.py b/tests/test_connector.py index ddcf67c574b..8aaf6d0445f 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -673,7 +673,7 @@ def test_release_not_started(loop): # assert conn._conns == {1: [(proto, 10)]} rec = conn._conns[1] assert rec[0][0] == proto - assert rec[0][1] == pytest.approx(loop.time(), abs=0.01) + assert rec[0][1] == pytest.approx(loop.time(), abs=0.05) assert not proto.close.called conn.close() From 5acaa94a572a06dcd3a0251b209dfd90374d47cf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 14:19:55 +0200 Subject: [PATCH 16/33] Bump to 2.3.2 --- CHANGES.rst | 15 +++++++++++++++ CHANGES/2385.bugfix | 1 - CHANGES/2408.bugfix | 1 - CHANGES/2414.bugfix | 1 - CHANGES/2423.bugfix | 1 - CHANGES/2424.bugfix | 1 - CHANGES/2431.bugfix | 1 - CHANGES/2441.bugfix | 1 - CHANGES/2451.doc | 1 - aiohttp/__init__.py | 2 +- 10 files changed, 16 insertions(+), 9 deletions(-) delete mode 100644 CHANGES/2385.bugfix delete mode 100644 CHANGES/2408.bugfix delete mode 100644 CHANGES/2414.bugfix delete mode 100644 CHANGES/2423.bugfix delete mode 100644 CHANGES/2424.bugfix delete mode 100644 CHANGES/2431.bugfix delete mode 100644 CHANGES/2441.bugfix delete mode 100644 CHANGES/2451.doc diff --git a/CHANGES.rst b/CHANGES.rst index b45122c6af7..3e31d059b34 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,21 @@ Changelog .. towncrier release notes start +2.3.2 (2017-11-01) +================== + +- Fix passing client max size on cloning request obj. (#2385) +- Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy + connector. (#2408) +- Drop generated `_http_parser` shared object from tarball distribution. (#2414) +- Fix connector convert OSError to ClientConnectorError. (#2423) +- Fix connection attempts for multiple dns hosts. (#2424) +- Fix ValueError for AF_INET6 sockets if a preexisting INET6 socket to the + `aiohttp.web.run_app` function. (#2431) +- _SessionRequestContextManager closes the session properly now. (#2441) +- Rename `from_env` to `trust_env` in client reference. (#2451) + + 2.3.1 (2017-10-18) ================== diff --git a/CHANGES/2385.bugfix b/CHANGES/2385.bugfix deleted file mode 100644 index fdc68de8f54..00000000000 --- a/CHANGES/2385.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix passing client max size on cloning request obj. diff --git a/CHANGES/2408.bugfix b/CHANGES/2408.bugfix deleted file mode 100644 index 4fec0c8b07c..00000000000 --- a/CHANGES/2408.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy connector diff --git a/CHANGES/2414.bugfix b/CHANGES/2414.bugfix deleted file mode 100644 index e7bd95f7927..00000000000 --- a/CHANGES/2414.bugfix +++ /dev/null @@ -1 +0,0 @@ -Drop generated `_http_parser` shared object from tarball distribution diff --git a/CHANGES/2423.bugfix b/CHANGES/2423.bugfix deleted file mode 100644 index 08823f4f69f..00000000000 --- a/CHANGES/2423.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix connector convert OSError to ClientConnectorError diff --git a/CHANGES/2424.bugfix b/CHANGES/2424.bugfix deleted file mode 100644 index f06477b975d..00000000000 --- a/CHANGES/2424.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix connection attempts for multiple dns hosts diff --git a/CHANGES/2431.bugfix b/CHANGES/2431.bugfix deleted file mode 100644 index 99e5c17db92..00000000000 --- a/CHANGES/2431.bugfix +++ /dev/null @@ -1 +0,0 @@ -fixed ValueError for AF_INET6 sockets if a preexisting INET6 socket to the `aiohttp.web.run_app` function. diff --git a/CHANGES/2441.bugfix b/CHANGES/2441.bugfix deleted file mode 100644 index 86f518b4ed4..00000000000 --- a/CHANGES/2441.bugfix +++ /dev/null @@ -1 +0,0 @@ -_SessionRequestContextManager closes the session properly now. diff --git a/CHANGES/2451.doc b/CHANGES/2451.doc deleted file mode 100644 index 386917d1926..00000000000 --- a/CHANGES/2451.doc +++ /dev/null @@ -1 +0,0 @@ -Rename `from_env` to `trust_env` in client reference. diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 9d8552410df..b18e3c50384 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.2b3' +__version__ = '2.3.2' # This relies on each of the submodules having an __all__ variable. From 8fd7928e58416d7ad985d27f0f48b4d072c89cc5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 14:30:33 +0200 Subject: [PATCH 17/33] Fix spelling errors --- CHANGES.rst | 2 +- docs/spelling_wordlist.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3e31d059b34..cfa633c2494 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,7 +25,7 @@ Changelog - Fix connection attempts for multiple dns hosts. (#2424) - Fix ValueError for AF_INET6 sockets if a preexisting INET6 socket to the `aiohttp.web.run_app` function. (#2431) -- _SessionRequestContextManager closes the session properly now. (#2441) +- `_SessionRequestContextManager` closes the session properly now. (#2441) - Rename `from_env` to `trust_env` in client reference. (#2451) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 5386a701c5e..9dc2509c13a 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -239,6 +239,7 @@ symlinks syscall syscalls Systemd +tarball TCP teardown Teardown From caab965687695c2c18a2bea20b0cf1d57979513f Mon Sep 17 00:00:00 2001 From: Victor K Date: Wed, 1 Nov 2017 16:31:22 +0200 Subject: [PATCH 18/33] Merge master to 2.3 (#2456) * Fix wrap oserrors. (#2442) * Fix wrap oserrors. * Skip unix socket tests on windows * Fix wrap ssl errors for proxy connector. (#2446) * Fix 2451: Rename from_env to trust_env in client reference. * Fix connection for multiple dns hosts (#2447) * Proof of concept fix for multiple dns hosts * Revert getpeercert * Added test for multiple dns hosts and errors. * Added change notes. * Remove redunant assert. * Minor docs formatting * Move TOC to separate RST file * Build tarball on appveyor * Bump to 2.3.2b1 * @asyncio.coroutine -> async * Update index.rst * Update toc.rst --- .appveyor.yml | 2 +- CHANGES.rst | 6 +- CHANGES/2408.bugfix | 1 + CHANGES/2423.bugfix | 1 + CHANGES/2424.bugfix | 1 + CHANGES/2451.doc | 1 + aiohttp/__init__.py | 2 +- aiohttp/connector.py | 113 ++++++++++++++++++----------- docs/client_reference.rst | 2 +- docs/conf.py | 2 +- docs/deployment.rst | 6 +- docs/index.rst | 63 +++++----------- docs/toc.rst | 36 ++++++++++ tests/test_connector.py | 147 +++++++++++++++++++++++++++++++++++++- tests/test_proxy.py | 131 ++++++++++++++++++++++++++------- 15 files changed, 390 insertions(+), 124 deletions(-) create mode 100644 CHANGES/2408.bugfix create mode 100644 CHANGES/2423.bugfix create mode 100644 CHANGES/2424.bugfix create mode 100644 CHANGES/2451.doc create mode 100644 docs/toc.rst diff --git a/.appveyor.yml b/.appveyor.yml index f07fba762fd..4abddac4c4c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,7 +17,7 @@ test_script: - "tools/build.cmd %PYTHON%\\python.exe setup.py test" after_test: - - "tools/build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + - "tools/build.cmd %PYTHON%\\python.exe setup.py sdist bdist_wheel" artifacts: - path: dist\* diff --git a/CHANGES.rst b/CHANGES.rst index 17ceb3f5280..b45122c6af7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,6 @@ -======= -Changes -======= +========= +Changelog +========= .. You should *NOT* be adding new change log entries to this file, this diff --git a/CHANGES/2408.bugfix b/CHANGES/2408.bugfix new file mode 100644 index 00000000000..4fec0c8b07c --- /dev/null +++ b/CHANGES/2408.bugfix @@ -0,0 +1 @@ +Fix ClientConnectorSSLError and ClientProxyConnectionError for proxy connector diff --git a/CHANGES/2423.bugfix b/CHANGES/2423.bugfix new file mode 100644 index 00000000000..08823f4f69f --- /dev/null +++ b/CHANGES/2423.bugfix @@ -0,0 +1 @@ +Fix connector convert OSError to ClientConnectorError diff --git a/CHANGES/2424.bugfix b/CHANGES/2424.bugfix new file mode 100644 index 00000000000..f06477b975d --- /dev/null +++ b/CHANGES/2424.bugfix @@ -0,0 +1 @@ +Fix connection attempts for multiple dns hosts diff --git a/CHANGES/2451.doc b/CHANGES/2451.doc new file mode 100644 index 00000000000..386917d1926 --- /dev/null +++ b/CHANGES/2451.doc @@ -0,0 +1 @@ +Rename `from_env` to `trust_env` in client reference. diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 21a8d4cb8e7..602f9cfdcd0 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.3.1' +__version__ = '2.3.2b1' # This relies on each of the submodules having an __all__ variable. diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 8b0ef4bbd1d..a74015e5f28 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -773,49 +773,73 @@ def _get_fingerprint_and_hashfunc(self, req): else: return (None, None) - async def _create_direct_connection(self, req): + async def _wrap_create_connection(self, *args, + req, client_error=ClientConnectorError, + **kwargs): + try: + return await self._loop.create_connection(*args, **kwargs) + except certificate_errors as exc: + raise ClientConnectorCertificateError( + req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + raise client_error(req.connection_key, exc) from exc + + async def _create_direct_connection(self, req, + *, client_error=ClientConnectorError): sslcontext = self._get_ssl_context(req) fingerprint, hashfunc = self._get_fingerprint_and_hashfunc(req) - hosts = await self._resolve_host(req.url.raw_host, req.port) + try: + hosts = await self._resolve_host(req.url.raw_host, req.port) + except OSError as exc: + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorError(req.connection_key, exc) from exc + + last_exc = None for hinfo in hosts: + host = hinfo['host'] + port = hinfo['port'] + try: - host = hinfo['host'] - port = hinfo['port'] - transp, proto = await self._loop.create_connection( + transp, proto = await self._wrap_create_connection( self._factory, host, port, ssl=sslcontext, family=hinfo['family'], proto=hinfo['proto'], flags=hinfo['flags'], server_hostname=hinfo['hostname'] if sslcontext else None, - local_addr=self._local_addr) - has_cert = transp.get_extra_info('sslcontext') - if has_cert and fingerprint: - sock = transp.get_extra_info('socket') - if not hasattr(sock, 'getpeercert'): - # Workaround for asyncio 3.5.0 - # Starting from 3.5.1 version - # there is 'ssl_object' extra info in transport - sock = transp._ssl_protocol._sslpipe.ssl_object - # gives DER-encoded cert as a sequence of bytes (or None) - cert = sock.getpeercert(binary_form=True) - assert cert - got = hashfunc(cert).digest() - expected = fingerprint - if got != expected: - transp.close() - if not self._cleanup_closed_disabled: - self._cleanup_closed_transports.append(transp) - raise ServerFingerprintMismatch( - expected, got, host, port) - return transp, proto - except certificate_errors as exc: - raise ClientConnectorCertificateError( - req.connection_key, exc) from exc - except ssl_errors as exc: - raise ClientConnectorSSLError(req.connection_key, exc) from exc - except OSError as exc: - raise ClientConnectorError(req.connection_key, exc) from exc + local_addr=self._local_addr, + req=req, client_error=client_error) + except ClientConnectorError as exc: + last_exc = exc + continue + + has_cert = transp.get_extra_info('sslcontext') + if has_cert and fingerprint: + sock = transp.get_extra_info('socket') + if not hasattr(sock, 'getpeercert'): + # Workaround for asyncio 3.5.0 + # Starting from 3.5.1 version + # there is 'ssl_object' extra info in transport + sock = transp._ssl_protocol._sslpipe.ssl_object + # gives DER-encoded cert as a sequence of bytes (or None) + cert = sock.getpeercert(binary_form=True) + assert cert + got = hashfunc(cert).digest() + expected = fingerprint + if got != expected: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + last_exc = ServerFingerprintMismatch( + expected, got, host, port) + continue + + return transp, proto + else: + raise last_exc async def _create_proxy_connection(self, req): headers = {} @@ -831,12 +855,10 @@ async def _create_proxy_connection(self, req): verify_ssl=req.verify_ssl, fingerprint=req.fingerprint, ssl_context=req.ssl_context) - try: - # create connection to proxy server - transport, proto = await self._create_direct_connection( - proxy_req) - except OSError as exc: - raise ClientProxyConnectionError(proxy_req, exc) from exc + + # create connection to proxy server + transport, proto = await self._create_direct_connection( + proxy_req, client_error=ClientProxyConnectionError) auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) if auth is not None: @@ -887,9 +909,10 @@ async def _create_proxy_connection(self, req): finally: transport.close() - transport, proto = await self._loop.create_connection( + transport, proto = await self._wrap_create_connection( self._factory, ssl=sslcontext, sock=rawsock, - server_hostname=req.host) + server_hostname=req.host, + req=req) finally: proxy_resp.close() @@ -921,6 +944,10 @@ def path(self): return self._path async def _create_connection(self, req): - _, proto = await self._loop.create_unix_connection( - self._factory, self._path) + try: + _, proto = await self._loop.create_unix_connection( + self._factory, self._path) + except OSError as exc: + raise ClientConnectorError(req.connection_key, exc) from exc + return proto diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 101aa3163d7..8f54b6391c2 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -143,7 +143,7 @@ The client session supports the context manager protocol for self closing. .. versionadded:: 2.3 - :param bool from_env: Get proxies information from *HTTP_PROXY* / + :param bool trust_env: Get proxies information from *HTTP_PROXY* / *HTTPS_PROXY* environment variables if the parameter is ``True`` (``False`` by default). diff --git a/docs/conf.py b/docs/conf.py index 1e704f315c5..ae54c9490ff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,7 +87,7 @@ # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = 'toc' # General information about the project. project = 'aiohttp' diff --git a/docs/deployment.rst b/docs/deployment.rst index 0acab53ef13..738a488a00c 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,8 +1,8 @@ .. _aiohttp-deployment: -========================= -aiohttp server deployment -========================= +================= +Server Deployment +================= There are several options for aiohttp server deployment: diff --git a/docs/index.rst b/docs/index.rst index ff82bc2dad9..bf7ae319fbf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,17 +3,18 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -aiohttp: Asynchronous HTTP Client/Server -======================================== +=============================================================== +aiohttp: Asynchronous HTTP Client/Server for Python and asyncio +=============================================================== -HTTP client/server for :term:`asyncio` (:pep:`3156`). +HTTP client/server for :term:`asyncio` and Python. .. _GitHub: https://github.com/aio-libs/aiohttp .. _Freenode: http://freenode.net Key Features ------------- +============ - Supports both :ref:`aiohttp-client` and :ref:`HTTP Server `. - Supports both :ref:`Server WebSockets ` and @@ -22,7 +23,7 @@ Key Features :ref:`aiohttp-web-signals` and pluggable routing. Library Installation --------------------- +==================== .. code-block:: bash @@ -44,7 +45,7 @@ This option is highly recommended: $ pip install aiodns Getting Started ---------------- +=============== Client example:: @@ -82,13 +83,13 @@ Server example:: Tutorial --------- +======== :ref:`Polls tutorial ` Source code ------------ +=========== The project is hosted on GitHub_ @@ -101,7 +102,7 @@ Continuous Integration. Dependencies ------------- +============ - Python 3.4.2+ - *chardet* @@ -126,7 +127,7 @@ Dependencies Communication channels ----------------------- +====================== *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs @@ -139,14 +140,14 @@ We support `Stack Overflow Please add *aiohttp* tag to your question there. Contributing ------------- +============ Please read the :ref:`instructions for contributors` before making a Pull Request. Authors and License -------------------- +=================== The ``aiohttp`` package is written mostly by Nikolay Kim and Andrew Svetlov. @@ -158,7 +159,7 @@ Feel free to improve this package and send a pull request to GitHub_. .. _aiohttp-backward-compatibility-policy: Policy for Backward Incompatible Changes ----------------------------------------- +======================================== *aiohttp* keeps backward compatibility. @@ -176,35 +177,7 @@ solved without major API change, but we are working hard for keeping these changes as rare as possible. -Contents --------- - -.. toctree:: - - client - client_reference - tutorial - web - web_reference - web_lowlevel - abc - multipart - multipart_reference - streams - api - logging - testing - deployment - faq - external - essays - contributing - changes - glossary - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +Table Of Contents +================= + +To see the full table of contents open the :ref:`link `. diff --git a/docs/toc.rst b/docs/toc.rst new file mode 100644 index 00000000000..02b0de633ab --- /dev/null +++ b/docs/toc.rst @@ -0,0 +1,36 @@ +Table of Contents +================= + +.. toctree:: + :caption: Table of Contents + :name: mastertoc + + Introduction + client + client_reference + tutorial + web + web_reference + web_lowlevel + abc + multipart + multipart_reference + streams + api + logging + testing + deployment + faq + external + essays + contributing + changes + glossary + Sitemap + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tests/test_connector.py b/tests/test_connector.py index 8edb1c0cd12..761f3cf41be 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -2,6 +2,7 @@ import asyncio import gc +import hashlib import os.path import platform import shutil @@ -9,6 +10,7 @@ import ssl import tempfile import unittest +import uuid from unittest import mock import pytest @@ -18,7 +20,7 @@ from aiohttp import client, web from aiohttp.client import ClientRequest from aiohttp.connector import Connection, _DNSCacheTable -from aiohttp.test_utils import unused_port +from aiohttp.test_utils import make_mocked_coro, unused_port @pytest.fixture() @@ -358,6 +360,108 @@ async def certificate_error(*args, **kwargs): '[CertificateError: ()]') +async def test_tcp_connector_multiple_hosts_errors(loop): + conn = aiohttp.TCPConnector(loop=loop) + + ip1 = '192.168.1.1' + ip2 = '192.168.1.2' + ip3 = '192.168.1.3' + ip4 = '192.168.1.4' + ip5 = '192.168.1.5' + ips = [ip1, ip2, ip3, ip4, ip5] + ips_tried = [] + + fingerprint = hashlib.sha256(b'foo').digest() + + req = ClientRequest('GET', URL('https://mocked.host'), + fingerprint=fingerprint, + loop=loop) + + async def _resolve_host(host, port): + return [{ + 'hostname': host, + 'host': ip, + 'port': port, + 'family': socket.AF_INET, + 'proto': 0, + 'flags': socket.AI_NUMERICHOST} + for ip in ips] + + conn._resolve_host = _resolve_host + + os_error = certificate_error = ssl_error = fingerprint_error = False + connected = False + + async def create_connection(*args, **kwargs): + nonlocal os_error, certificate_error, ssl_error, fingerprint_error + nonlocal connected + + ip = args[1] + + ips_tried.append(ip) + + if ip == ip1: + os_error = True + raise OSError + + if ip == ip2: + certificate_error = True + raise ssl.CertificateError + + if ip == ip3: + ssl_error = True + raise ssl.SSLError + + if ip == ip4: + fingerprint_error = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'not foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + if ip == ip5: + connected = True + tr, pr = mock.Mock(), None + + def get_extra_info(param): + if param == 'sslcontext': + return True + + if param == 'socket': + s = mock.Mock() + s.getpeercert.return_value = b'foo' + return s + + assert False + + tr.get_extra_info = get_extra_info + return tr, pr + + assert False + + conn._loop.create_connection = create_connection + + await conn.connect(req) + assert ips == ips_tried + + assert os_error + assert certificate_error + assert ssl_error + assert fingerprint_error + assert connected + + async def test_tcp_connector_resolve_host(loop): conn = aiohttp.TCPConnector(loop=loop, use_dns_cache=True) @@ -495,6 +599,19 @@ async def test_tcp_connector_dns_throttle_requests_cancelled_when_close( await f +def test_dns_error(loop): + connector = aiohttp.TCPConnector(loop=loop) + connector._resolve_host = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_get_pop_empty_conns(loop): # see issue #473 conn = aiohttp.BaseConnector(loop=loop) @@ -1247,6 +1364,34 @@ async def handler(request): assert r.status == 200 +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_not_found(loop): + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + +@pytest.mark.skipif(not hasattr(socket, 'AF_UNIX'), + reason="requires unix socket") +def test_unix_connector_permission(loop): + loop.create_unix_connection = make_mocked_coro( + raise_exception=PermissionError()) + connector = aiohttp.UnixConnector('/' + uuid.uuid4().hex, loop=loop) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + loop=loop, + ) + with pytest.raises(aiohttp.ClientConnectorError): + loop.run_until_complete(connector.connect(req)) + + def test_default_use_dns_cache(loop): conn = aiohttp.TCPConnector(loop=loop) assert conn.use_dns_cache diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 9a0380f520f..cc066904e2b 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -2,6 +2,7 @@ import gc import hashlib import socket +import ssl import unittest from unittest import mock @@ -215,7 +216,7 @@ def _test_connect_request_with_unicode_host(self, Request_mock): Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80", mock.ANY, loop=loop) - def test_proxy_connection_error(self): + def test_proxy_dns_error(self): connector = aiohttp.TCPConnector(loop=self.loop) connector._resolve_host = make_mocked_coro( raise_exception=OSError('dont take it serious')) @@ -231,6 +232,24 @@ def test_proxy_connection_error(self): self.assertEqual(req.url.path, '/') self.assertEqual(dict(req.headers), expected_headers) + def test_proxy_connection_error(self): + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro([{ + 'hostname': 'www.python.org', + 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, + 'flags': socket.AI_NUMERICHOST}]) + connector._loop.create_connection = make_mocked_coro( + raise_exception=OSError('dont take it serious')) + + req = ClientRequest( + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientProxyConnectionError): + self.loop.run_until_complete(connector.connect(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_auth(self, ClientRequestMock): proxy_req = ClientRequest( @@ -314,30 +333,6 @@ def test_auth_from_url(self, ClientRequestMock): ssl_context=None, verify_ssl=None) conn.close() - @mock.patch('aiohttp.connector.ClientRequest') - def test_auth__not_modifying_request(self, ClientRequestMock): - proxy_req = ClientRequest('GET', - URL('http://user:pass@proxy.example.com'), - loop=self.loop) - ClientRequestMock.return_value = proxy_req - proxy_req_headers = dict(proxy_req.headers) - - connector = aiohttp.TCPConnector(loop=self.loop) - connector._resolve_host = make_mocked_coro( - raise_exception=OSError('nothing personal')) - - req = ClientRequest( - 'GET', URL('http://www.python.org'), - proxy=URL('http://user:pass@proxy.example.com'), - loop=self.loop, - ) - req_headers = dict(req.headers) - with self.assertRaises(aiohttp.ClientConnectorError): - self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.headers, req_headers) - self.assertEqual(req.url.path, '/') - self.assertEqual(proxy_req.headers, proxy_req_headers) - @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), @@ -375,6 +370,92 @@ def test_https_connect(self, ClientRequestMock): proxy_resp.close() self.loop.run_until_complete(req.close()) + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_certificate_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.CertificateError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorCertificateError): + self.loop.run_until_complete(connector._create_connection(req)) + + @mock.patch('aiohttp.connector.ClientRequest') + def test_https_connect_ssl_error(self, ClientRequestMock): + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), + loop=self.loop) + ClientRequestMock.return_value = proxy_req + + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) + proxy_resp._loop = self.loop + proxy_req.send = send_mock = mock.Mock() + send_mock.return_value = proxy_resp + proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) + + connector = aiohttp.TCPConnector(loop=self.loop) + connector._resolve_host = make_mocked_coro( + [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80, + 'family': socket.AF_INET, 'proto': 0, 'flags': 0}]) + + seq = 0 + + @asyncio.coroutine + def create_connection(*args, **kwargs): + nonlocal seq + seq += 1 + + # connection to http://proxy.example.com + if seq == 1: + return mock.Mock(), mock.Mock() + # connection to https://www.python.org + elif seq == 2: + raise ssl.SSLError + else: + assert False + + self.loop.create_connection = create_connection + + req = ClientRequest( + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), + loop=self.loop, + ) + with self.assertRaises(aiohttp.ClientConnectorSSLError): + self.loop.run_until_complete(connector._create_connection(req)) + @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_runtime_error(self, ClientRequestMock): proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), From e740a1d64b919470c5edbf616e751ec6d283f75f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 16:56:13 +0200 Subject: [PATCH 19/33] Don't drop aiohttp manylinux from foreign architecture on docker run --- tools/build-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 09eeb242d3d..6caf6249ade 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -44,7 +44,7 @@ rm -fv /io/dist/*-linux_*.whl echo echo echo "Cleanup non-$package_name wheels" -find /io/dist -maxdepth 1 -type f ! -name "$package_name"'-*-manylinux1_'"$arch"'.whl' -print0 | xargs -0 rm -rf +find /io/dist -maxdepth 1 -type f ! -name "$package_name"'-*-manylinux1_*.whl' -print0 | xargs -0 rm -rf echo echo From 32e577e4070219c29a6fef0f866951c024ae76c8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 18:39:41 +0200 Subject: [PATCH 20/33] Don't build python 3.4 binary wheels --- tools/build-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 6caf6249ade..878a437aa65 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -6,7 +6,7 @@ fi set -euo pipefail # ref: https://coderwall.com/p/fkfaqq/safer-bash-scripts-with-set-euxo-pipefail -PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +PYTHON_VERSIONS="cp35-cp35m cp36-cp36m" # Avoid creation of __pycache__/*.py[c|o] export PYTHONDONTWRITEBYTECODE=1 From dc836413a867f89d72260bd8c50ecd20689733e3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 1 Nov 2017 19:15:38 +0200 Subject: [PATCH 21/33] Clarify what kind of issue (server or client) in the template --- .github/ISSUE_TEMPLATE.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 319163f92a5..a6b60e79706 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -19,5 +19,10 @@ ## Your environment + This includes aiohttp version, OS, proxy server and other bits that + are related to your case. + + IMPORTANT: aiohttp is both server framework and client library. + For getting rid of confusing please put 'server', 'client' or 'both' + word here. + --> From 5f2c4cb56f777f01d6ee8b3e8a60926083738b0b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 2 Nov 2017 10:00:54 +0200 Subject: [PATCH 22/33] Compile Cython files before making tarball on Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3f10c58df34..c464dff9a97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -212,6 +212,7 @@ jobs: - docker script: - ./tools/run_docker.sh "aiohttp" + - pip install -r requirements/ci.txt # to compile *.c files by Cython deploy: provider: pypi # `skip_cleanup: true` is required to preserve binary wheels, built From 9b85749b4ead3a12f23a00a3eded1b92e914d606 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 11:05:00 +0300 Subject: [PATCH 23/33] #2409: Get rid of couple warnings --- tests/test_client_request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_client_request.py b/tests/test_client_request.py index d78368c87c9..0574d9badda 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -1027,6 +1027,8 @@ def test_terminate_with_closed_loop(loop, conn): assert req._writer is not None writer = req._writer = mock.Mock() + loop.run_until_complete(asyncio.sleep(0.05, loop=loop)) + loop.close() req.terminate() assert req._writer is None From 04f59e7907b5b97f4ac6ae1c06781b9858691f82 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 11:51:35 +0300 Subject: [PATCH 24/33] Fix #2409: Get rid of warnings from tests --- tests/test_worker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index 1134eb66542..9fa7144df17 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -97,9 +97,10 @@ def test_run_wsgi(worker, loop): assert loop.is_closed() -def test_handle_quit(worker): +def test_handle_quit(worker, loop): with mock.patch('asyncio.ensure_future') as m_ensure_future: worker.loop = mock.Mock() + worker.close = mock.Mock() worker.handle_quit(object(), object()) assert not worker.alive assert worker.exit_code == 0 @@ -185,7 +186,7 @@ def test__get_valid_log_format_exc(worker): assert '%(name)s' in str(exc) -async def test__run_ok(worker, loop): +async def test__run_ok_tcp(worker, loop): skip_if_no_dict(loop) worker.ppid = 1 From 1049c865a08ba549cd193185a8aebcd20232ff8b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 13:32:54 +0200 Subject: [PATCH 25/33] Multipart doc (#2459) * Convert a part of multipart's doc * Fix #2395: get rid of autodoc in multipart. * Fix typo --- aiohttp/multipart.py | 112 +++++++----------------- docs/api.rst | 9 -- docs/multipart_reference.rst | 162 ++++++++++++++++++++++++++++++++++- 3 files changed, 192 insertions(+), 91 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 3cacf46e356..e28932313aa 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -162,8 +162,11 @@ def content_disposition_filename(params, name='filename'): class MultipartResponseWrapper(object): - """Wrapper around the :class:`MultipartBodyReader` to take care about - underlying connection and close it when it needs in.""" + """Wrapper around the MultipartBodyReader. + + It takes care about + underlying connection and close it when it needs in. + """ def __init__(self, resp, stream): self.resp = resp @@ -180,10 +183,7 @@ def __anext__(self): return part def at_eof(self): - """Returns ``True`` when all response data had been read. - - :rtype: bool - """ + """Returns True when all response data had been read.""" return self.resp.content.at_eof() async def next(self): @@ -238,11 +238,9 @@ def next(self): def read(self, *, decode=False): """Reads body part data. - :param bool decode: Decodes data following by encoding - method from `Content-Encoding` header. If it missed - data remains untouched - - :rtype: bytearray + decode: Decodes data following by encoding + method from Content-Encoding header. If it missed + data remains untouched """ if self._at_eof: return b'' @@ -257,9 +255,7 @@ def read(self, *, decode=False): def read_chunk(self, size=chunk_size): """Reads body part content chunk of the specified size. - :param int size: chunk size - - :rtype: bytearray + size: chunk size """ if self._at_eof: return b'' @@ -278,13 +274,8 @@ def read_chunk(self, size=chunk_size): return chunk async def _read_chunk_from_length(self, size): - """Reads body part content chunk of the specified size. - The body part must has `Content-Length` header with proper value. - - :param int size: chunk size - - :rtype: bytearray - """ + # Reads body part content chunk of the specified size. + # The body part must has Content-Length header with proper value. assert self._length is not None, \ 'Content-Length required for chunked read' chunk_size = min(size, self._length - self._read_bytes) @@ -292,13 +283,8 @@ async def _read_chunk_from_length(self, size): return chunk async def _read_chunk_from_stream(self, size): - """Reads content chunk of body part with unknown length. - The `Content-Length` header for body part is not necessary. - - :param int size: chunk size - - :rtype: bytearray - """ + # Reads content chunk of body part with unknown length. + # The Content-Length header for body part is not necessary. assert size >= len(self._boundary) + 2, \ 'Chunk size must be greater or equal than boundary length + 2' first_chunk = self._prev_chunk is None @@ -328,10 +314,7 @@ async def _read_chunk_from_stream(self, size): @asyncio.coroutine def readline(self): - """Reads body part by line by line. - - :rtype: bytearray - """ + """Reads body part by line by line.""" if self._at_eof: return b'' @@ -361,10 +344,7 @@ def readline(self): @asyncio.coroutine def release(self): - """Like :meth:`read`, but reads all the data to the void. - - :rtype: None - """ + """Like read(), but reads all the data to the void.""" if self._at_eof: return while not self._at_eof: @@ -372,13 +352,7 @@ def release(self): @asyncio.coroutine def text(self, *, encoding=None): - """Like :meth:`read`, but assumes that body part contains text data. - - :param str encoding: Custom text encoding. Overrides specified - in charset param of `Content-Type` header - - :rtype: str - """ + """Like read(), but assumes that body part contains text data.""" data = yield from self.read(decode=True) # see https://www.w3.org/TR/html5/forms.html#multipart/form-data-encoding-algorithm # NOQA # and https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#dom-xmlhttprequest-send # NOQA @@ -387,11 +361,7 @@ def text(self, *, encoding=None): @asyncio.coroutine def json(self, *, encoding=None): - """Like :meth:`read`, but assumes that body parts contains JSON data. - - :param str encoding: Custom JSON encoding. Overrides specified - in charset param of `Content-Type` header - """ + """Like read(), but assumes that body parts contains JSON data.""" data = yield from self.read(decode=True) if not data: return None @@ -400,11 +370,8 @@ def json(self, *, encoding=None): @asyncio.coroutine def form(self, *, encoding=None): - """Like :meth:`read`, but assumes that body parts contains form + """Like read(), but assumes that body parts contains form urlencoded data. - - :param str encoding: Custom form encoding. Overrides specified - in charset param of `Content-Type` header """ data = yield from self.read(decode=True) if not data: @@ -415,28 +382,12 @@ def form(self, *, encoding=None): encoding=encoding) def at_eof(self): - """Returns ``True`` if the boundary was reached or - ``False`` otherwise. - - :rtype: bool - """ + """Returns True if the boundary was reached or False otherwise.""" return self._at_eof def decode(self, data): - """Decodes data according the specified `Content-Encoding` - or `Content-Transfer-Encoding` headers value. - - Supports ``gzip``, ``deflate`` and ``identity`` encodings for - `Content-Encoding` header. - - Supports ``base64``, ``quoted-printable``, ``binary`` encodings for - `Content-Transfer-Encoding` header. - - :param bytearray data: Data to decode. - - :raises: :exc:`RuntimeError` - if encoding is unknown. - - :rtype: bytes + """Decodes data according the specified Content-Encoding + or Content-Transfer-Encoding headers value. """ if CONTENT_TRANSFER_ENCODING in self.headers: data = self._decode_content_transfer(data) @@ -470,24 +421,25 @@ def _decode_content_transfer(self, data): ''.format(encoding)) def get_charset(self, default=None): - """Returns charset parameter from ``Content-Type`` header or default. - """ + """Returns charset parameter from Content-Type header or default.""" ctype = self.headers.get(CONTENT_TYPE, '') mimetype = parse_mimetype(ctype) return mimetype.parameters.get('charset', default) @reify def name(self): - """Returns filename specified in Content-Disposition header or ``None`` - if missed or header is malformed.""" + """Returns name specified in Content-Disposition header or None + if missed or header is malformed. + """ _, params = parse_content_disposition( self.headers.get(CONTENT_DISPOSITION)) return content_disposition_filename(params, 'name') @reify def filename(self): - """Returns filename specified in Content-Disposition header or ``None`` - if missed or header is malformed.""" + """Returns filename specified in Content-Disposition header or None + if missed or header is malformed. + """ _, params = parse_content_disposition( self.headers.get(CONTENT_DISPOSITION)) return content_disposition_filename(params, 'filename') @@ -557,10 +509,8 @@ def from_response(cls, response): return obj def at_eof(self): - """Returns ``True`` if the final boundary was reached or - ``False`` otherwise. - - :rtype: bool + """Returns True if the final boundary was reached or + False otherwise. """ return self._at_eof diff --git a/docs/api.rst b/docs/api.rst index fe0a53ea13e..3168631fbbc 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -210,12 +210,3 @@ For concrete usage examples see :ref:`signals in aiohttp.web .. method:: freeze() Freeze the list. After the call any content modification is forbidden. - - -aiohttp.multipart module ------------------------- - -.. automodule:: aiohttp.multipart - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/multipart_reference.rst b/docs/multipart_reference.rst index 201e0129202..18d99b57bed 100644 --- a/docs/multipart_reference.rst +++ b/docs/multipart_reference.rst @@ -3,7 +3,7 @@ Multipart reference =================== -.. class:: MultipartResponseWrapper +.. class:: MultipartResponseWrapper(resp, stream) Wrapper around the :class:`MultipartBodyReader` to take care about underlying connection and close it when it needs in. @@ -24,3 +24,163 @@ Multipart reference Releases the connection gracefully, reading all the content to the void. + +.. class:: BodyPartReader(boundary, headers, content) + + Multipart reader for single body part. + + .. comethod:: read(*, decode=False) + + Reads body part data. + + :param bool decode: Decodes data following by encoding method + from ``Content-Encoding`` header. If it + missed data remains untouched + + :rtype: bytearray + + .. comethod:: read_chunk(size=chunk_size) + + Reads body part content chunk of the specified size. + + :param int size: chunk size + + :rtype: bytearray + + .. comethod:: readline() + + Reads body part by line by line. + + :rtype: bytearray + + .. comethod:: release() + + Like :meth:`read`, but reads all the data to the void. + + :rtype: None + + .. comethod:: text(*, encoding=None) + + Like :meth:`read`, but assumes that body part contains text data. + + :param str encoding: Custom text encoding. Overrides specified + in charset param of ``Content-Type`` header + + :rtype: str + + .. comethod:: json(*, encoding=None) + + Like :meth:`read`, but assumes that body parts contains JSON data. + + :param str encoding: Custom JSON encoding. Overrides specified + in charset param of ``Content-Type`` header + + .. comethod:: form(*, encoding=None) + + Like :meth:`read`, but assumes that body parts contains form + urlencoded data. + + :param str encoding: Custom form encoding. Overrides specified + in charset param of ``Content-Type`` header + + .. method:: at_eof() + + Returns ``True`` if the boundary was reached or ``False`` otherwise. + + :rtype: bool + + .. method:: decode(data) + + Decodes data according the specified ``Content-Encoding`` + or ``Content-Transfer-Encoding`` headers value. + + Supports ``gzip``, ``deflate`` and ``identity`` encodings for + ``Content-Encoding`` header. + + Supports ``base64``, ``quoted-printable``, ``binary`` encodings for + ``Content-Transfer-Encoding`` header. + + :param bytearray data: Data to decode. + + :raises: :exc:`RuntimeError` - if encoding is unknown. + + :rtype: bytes + + .. method:: get_charset(default=None) + + Returns charset parameter from ``Content-Type`` header or default. + + .. attribute:: name + + A field *name* specified in ``Content-Disposition`` header or ``None`` + if missed or header is malformed. + + Readonly :class:`str` property. + + .. attribute:: name + + A field *filename* specified in ``Content-Disposition`` header or ``None`` + if missed or header is malformed. + + Readonly :class:`str` property. + + +.. class:: MultipartReader(headers, content) + + Multipart body reader. + + .. classmethod:: from_response(cls, response) + + Constructs reader instance from HTTP response. + + :param response: :class:`~aiohttp.client.ClientResponse` instance + + .. method:: at_eof() + + Returns ``True`` if the final boundary was reached or + ``False`` otherwise. + + :rtype: bool + + .. comethod:: next() + + Emits the next multipart body part. + + .. comethod:: release() + + Reads all the body parts to the void till the final boundary. + + .. comethod:: fetch_next_part() + + Returns the next body part reader. + + +.. class:: MultipartWriter(subtype='mixed', boundary=None) + + Multipart body writer. + + .. attribute:: boundary + + .. method:: append(obj, headers=None) + + Append an object to writer. + + .. method:: append_payload(payload) + + Adds a new body part to multipart writer. + + .. method:: append_json(obj, headers=None) + + Helper to append JSON part. + + .. method:: append_form(obj, headers=None) + + Helper to append form urlencoded part. + + .. attribute:: size + + Size of the payload. + + .. comethod:: write(writer) + + Write body. From 5513dcd79ffefc17163e8808de145ec1b9fd3a03 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 15:22:50 +0300 Subject: [PATCH 26/33] Fix nonexisting domain issue on bad internet connection in client functional tests --- tests/test_client_functional.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index f7d6622c7bc..4187183b074 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -2226,7 +2226,17 @@ async def handler(request): async def test_close_detached_session_on_non_existing_addr(loop): - session = aiohttp.ClientSession(loop=loop) + class FakeResolver(AbstractResolver): + async def resolve(host, port=0, family=socket.AF_INET): + return {} + + async def close(self): + pass + + connector = aiohttp.TCPConnector(resolver=FakeResolver(), + loop=loop) + + session = aiohttp.ClientSession(connector=connector) async with session: cm = session.get('http://non-existing.example.com') From bc45fac791ad73fb476823b970e1d2b4b9fd5437 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 15:43:54 +0300 Subject: [PATCH 27/33] Fix #2421: cleanup web.Request attributes --- aiohttp/web_fileresponse.py | 4 ++-- aiohttp/web_request.py | 14 ++++++-------- aiohttp/web_response.py | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/aiohttp/web_fileresponse.py b/aiohttp/web_fileresponse.py index aecc5e9e2d9..2c7ccaa8e69 100644 --- a/aiohttp/web_fileresponse.py +++ b/aiohttp/web_fileresponse.py @@ -116,8 +116,8 @@ async def _sendfile_system(self, request, fobj, count): writer = await self._sendfile_fallback(request, fobj, count) else: writer = request._protocol.writer.replace( - request._writer, SendfilePayloadWriter) - request._writer = writer + request._payload_writer, SendfilePayloadWriter) + request._payload_writer = writer await super().prepare(request) await writer.sendfile(fobj, count) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 31dc9e17be5..4a07515b872 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -62,7 +62,7 @@ class BaseRequest(collections.MutableMapping, HeadersMixin): POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_TRACE, hdrs.METH_DELETE} - def __init__(self, message, payload, protocol, writer, task, + def __init__(self, message, payload, protocol, payload_writer, task, loop, *, client_max_size=1024**2, state=None, @@ -71,8 +71,7 @@ def __init__(self, message, payload, protocol, writer, task, state = {} self._message = message self._protocol = protocol - self._transport = protocol.transport - self._writer = writer + self._payload_writer = payload_writer self._payload = payload self._headers = message.headers @@ -133,7 +132,7 @@ def clone(self, *, method=sentinel, rel_url=sentinel, message, self._payload, self._protocol, - self._writer, + self._payload_writer, self._task, self._loop, client_max_size=self._client_max_size, @@ -154,7 +153,7 @@ def transport(self): @property def writer(self): - return self._writer + return self._payload_writer @property def message(self): @@ -261,7 +260,7 @@ def scheme(self): scheme = self._scheme if scheme is not None: return scheme - if self._transport.get_extra_info('sslcontext'): + if self.transport.get_extra_info('sslcontext'): return 'https' else: return 'http' @@ -316,8 +315,7 @@ def remote(self): remote = self._remote if remote is not None: return remote - transport = self._transport - peername = transport.get_extra_info('peername') + peername = self.transport.get_extra_info('peername') if isinstance(peername, (list, tuple)): return peername[0] else: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 1827ff31ebb..5dfd5203b9a 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -343,7 +343,7 @@ def _start(self, request, self._keep_alive = keep_alive version = request.version - writer = self._payload_writer = request._writer + writer = self._payload_writer = request._payload_writer headers = self._headers for cookie in self._cookies.values(): From de32b56e87f8bb7de415be6b065739f72c286907 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 3 Nov 2017 17:08:37 +0200 Subject: [PATCH 28/33] Drop deprecated on_loop_available signal (#2460) --- aiohttp/web.py | 11 +---------- tests/test_web_application.py | 11 ----------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/aiohttp/web.py b/aiohttp/web.py index cd397e487fc..8e63590b1b8 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -20,7 +20,7 @@ from .helpers import AccessLogger from .http import HttpVersion # noqa from .log import access_logger, web_logger -from .signals import FuncSignal, PostSignal, PreSignal, Signal +from .signals import PostSignal, PreSignal, Signal from .web_exceptions import * # noqa from .web_fileresponse import * # noqa from .web_middlewares import * # noqa @@ -74,7 +74,6 @@ def __init__(self, *, self._on_pre_signal = PreSignal() self._on_post_signal = PostSignal() - self._on_loop_available = FuncSignal(self) self._on_response_prepare = Signal(self) self._on_startup = Signal(self) self._on_shutdown = Signal(self) @@ -123,7 +122,6 @@ def _set_loop(self, loop): "web.Application instance initialized with different loop") self._loop = loop - self._on_loop_available.send(self) # set loop debug if self._debug is ...: @@ -144,7 +142,6 @@ def freeze(self): self._frozen = True self._middlewares = tuple(self._prepare_middleware()) self._router.freeze() - self._on_loop_available.freeze() self._on_pre_signal.freeze() self._on_post_signal.freeze() self._on_response_prepare.freeze() @@ -192,12 +189,6 @@ def add_subapp(self, prefix, subapp): subapp._set_loop(self._loop) return resource - @property - def on_loop_available(self): - warnings.warn("on_loop_available is deprecated and will be removed", - DeprecationWarning, stacklevel=2) - return self._on_loop_available - @property def on_response_prepare(self): return self._on_response_prepare diff --git a/tests/test_web_application.py b/tests/test_web_application.py index b29381831be..d50816403e2 100644 --- a/tests/test_web_application.py +++ b/tests/test_web_application.py @@ -46,17 +46,6 @@ def test_set_loop_with_different_loops(loop): app._set_loop(loop=object()) -def test_on_loop_available(loop): - app = web.Application() - - cb = mock.Mock() - with pytest.warns(DeprecationWarning): - app.on_loop_available.append(cb) - - app._set_loop(loop) - cb.assert_called_with(app) - - @pytest.mark.parametrize('debug', [True, False]) def test_app_make_handler_debug_exc(loop, mocker, debug): app = web.Application(debug=debug) From 092dc66d1466a6f053f729f44e4b9a721616f62b Mon Sep 17 00:00:00 2001 From: Zack Hsi Date: Sat, 4 Nov 2017 01:33:19 -0700 Subject: [PATCH 29/33] Document client error response fields (#2462) --- docs/client_reference.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 8f54b6391c2..151fa8db51b 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1576,6 +1576,18 @@ Response errors Instance of :class:`RequestInfo` object, contains information about request. + .. attribute:: code + + HTTP status code of response (:class:`int`), e.g. ``200``. + + .. attribute:: message + + Message of response (:class:`str`), e.g. ``"OK"``. + + .. attribute:: headers + + Headers in response, a list of pairs. + .. attribute:: history History from failed response, if available, else empty tuple. From 47188bbf95cb1311eb5c33f93d2bf554f2e09b85 Mon Sep 17 00:00:00 2001 From: Zack Hsi Date: Sat, 4 Nov 2017 01:33:19 -0700 Subject: [PATCH 30/33] Document client error response fields (#2462) --- docs/client_reference.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 8f54b6391c2..151fa8db51b 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1576,6 +1576,18 @@ Response errors Instance of :class:`RequestInfo` object, contains information about request. + .. attribute:: code + + HTTP status code of response (:class:`int`), e.g. ``200``. + + .. attribute:: message + + Message of response (:class:`str`), e.g. ``"OK"``. + + .. attribute:: headers + + Headers in response, a list of pairs. + .. attribute:: history History from failed response, if available, else empty tuple. From 3e6bc93bcbbf6882262010090552d4e881e33945 Mon Sep 17 00:00:00 2001 From: Zack Hsi Date: Sat, 4 Nov 2017 11:40:47 -0700 Subject: [PATCH 31/33] Update feature branch to reflect current versions (#2463) No longer version 0 :) --- .github/PULL_REQUEST_TEMPLATE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0c29da341c2..aa279007096 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,8 +5,9 @@ - If you propose a new feature or improvement, select "master" as a target branch; - - If this is a bug fix or documentation amendment, select - the latest release branch (which looks like "0.xx") --> + - If this is a bug fix or documentation amendment, select the latest + [release branch](https://github.com/aio-libs/aiohttp/branches) (which + looks like "x.x.x") --> ## What do these changes do? From 94656c0a9f93dd2b713141ea72fdb1fe858669af Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Sat, 4 Nov 2017 18:45:55 +0000 Subject: [PATCH 32/33] Clarify branch name in PR template Follow up #2464 --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index aa279007096..984e09c7f8b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ as a target branch; - If this is a bug fix or documentation amendment, select the latest [release branch](https://github.com/aio-libs/aiohttp/branches) (which - looks like "x.x.x") --> + looks like "X.Y" e.g. "2.3") --> ## What do these changes do? From 0b34a8f8e7f70bf8dd3d476f38b2ce8a3f50c4fa Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 6 Nov 2017 14:59:21 +0100 Subject: [PATCH 33/33] Scheduled weekly dependency update for week 45 (#2468) * Update coverage from 4.4.1 to 4.4.2 * Update cython from 0.27.2 to 0.27.3 * Update cython from 0.27.2 to 0.27.3 * Update multidict from 3.3.0 to 3.3.2 --- requirements/ci-wheel.txt | 6 +++--- requirements/wheel.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/ci-wheel.txt b/requirements/ci-wheel.txt index e13f040ed70..65ecfdbd806 100644 --- a/requirements/ci-wheel.txt +++ b/requirements/ci-wheel.txt @@ -2,12 +2,12 @@ async-timeout==2.0.0 brotlipy==0.7.0 cchardet==2.1.1 chardet==3.0.4 -coverage==4.4.1 -cython==0.27.2 +coverage==4.4.2 +cython==0.27.3 flake8==3.5.0 gunicorn==19.7.1 isort==4.2.15 -multidict==3.3.0 +multidict==3.3.2 pip==9.0.1 pyflakes==1.6.0 pytest-cov==2.5.1 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index 0c9df4ede62..3d1f2842567 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ -cython==0.27.2 +cython==0.27.3 pytest==3.2.3 twine==1.9.1