From 2297325962412113b412352a4d2c8a0f64b5bc71 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 22 Feb 2018 20:09:42 +0200 Subject: [PATCH 01/15] Prevent Windows absolute URLs in static files --- aiohttp/web_urldispatcher.py | 10 +++++++++- tests/test_web_sendfile_functional.py | 2 +- tests/test_web_urldispatcher.py | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 4f8def2e7f1..18eaac18170 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -552,14 +552,22 @@ def __iter__(self): return iter(self._routes.values()) async def _handle(self, request): - filename = request.match_info['filename'] + rel_url = request.match_info['filename'] try: + filename = Path(rel_url) + if filename.anchor: + # rel_url is an absolute name like + # /static/\\machine_name\c$ or /static/D:\path + # where the static dir is totally different + raise HTTPForbidden() filepath = self._directory.joinpath(filename).resolve() if not self._follow_symlinks: filepath.relative_to(self._directory) except (ValueError, FileNotFoundError) as error: # relatively safe raise HTTPNotFound() from error + except HTTPForbidden: + raise except Exception as error: # perm error or other kind! request.app.logger.exception(error) diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index f364ebe60e8..d7fa1a7346b 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -306,7 +306,7 @@ async def test_static_file_directory_traversal_attack(loop, aiohttp_client): url_abspath = \ '/static/' + os.path.abspath(os.path.join(dirname, relpath)) resp = await client.get(url_abspath) - assert 404 == resp.status + assert 403 == resp.status def test_static_route_path_existence_check(): diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py index 167719b0cc5..657165e712e 100644 --- a/tests/test_web_urldispatcher.py +++ b/tests/test_web_urldispatcher.py @@ -1,5 +1,6 @@ import functools import os +import pathlib import shutil import tempfile from unittest import mock @@ -274,7 +275,7 @@ async def test_access_special_resource(tmp_dir_path, aiohttp_client): # Request the root of the static directory. r = await client.get('/special') - assert r.status == 404 + assert r.status == 403 async def test_partialy_applied_handler(aiohttp_client): @@ -467,3 +468,17 @@ async def post(self): r = await client.put("/a") assert r.status == 405 await r.release() + + +async def test_static_absolute_url(aiohttp_client, tmpdir): + # requested url is an absolute name like + # /static/\\machine_name\c$ or /static/D:\path + # where the static dir is totally different + app = web.Application() + fname = tmpdir / 'file.txt' + fname.write_text('sample text', 'ascii') + here = pathlib.Path(__file__).parent + app.router.add_static('/static', here) + client = await aiohttp_client(app) + resp = await client.get('/static/' + str(fname)) + assert resp.status == 403 From a0aeee4e9ab8aa982d8944ca3fd633700a028e76 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 23 Feb 2018 10:46:13 +0200 Subject: [PATCH 02/15] Bump to 3.0.2 --- CHANGES.rst | 15 +++++++++++++++ aiohttp/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 17178046126..442a50c2673 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,21 @@ Changelog .. towncrier release notes start +3.0.2 (2018-02-23) +================== + +Security Fix +------------ + +- Prevent Windows absolute URLs in static files. Paths like + ``/static/D:\path`` and ``/static/\\hostname\drive\path`` are + forbidden. + +3.0.1 +===== + +- Technical release for fixing disrtribution problems. + 3.0.0 (2018-02-12) ================== diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 7ea8b38e600..9ad7e523876 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.1' +__version__ = '3.0.2' # This relies on each of the submodules having an __all__ variable. From 66858fc999f0cf318c286285a0e27435075a83bb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 23 Feb 2018 10:58:32 +0200 Subject: [PATCH 03/15] Fix spelling --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 442a50c2673..6cb7a03719e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,7 +27,7 @@ Security Fix 3.0.1 ===== -- Technical release for fixing disrtribution problems. +- Technical release for fixing distribution problems. 3.0.0 (2018-02-12) ================== From 68ecded1afb07a441d69dc8f8dcab820c120f82d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 24 Feb 2018 00:02:27 +0300 Subject: [PATCH 04/15] Fix typo in attr name --- aiohttp/web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 8b52de17ffb..c709cb291ec 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -251,7 +251,7 @@ def data_received(self, data): if self._waiter: self._waiter.set_result(None) - self._upgraded = upgraded + self._upgrade = upgraded if upgraded and tail: self._message_tail = tail From 72da093a95d1dd2dd90876d01deecc0b2d0bd1a6 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Feb 2018 10:33:40 +0300 Subject: [PATCH 05/15] Update Powered by list --- docs/powered_by.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index 5221080c9ba..9a969dd2965 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -14,11 +14,20 @@ make a Pull Request! * `Skyscanner Hotels `_ * `Ocean S.A. `_ * `GNS3 `_ -* `TutorCruncher socket `_ +* `TutorCruncher socket + `_ * `Morpheus messaging microservice `_ * `Eyepea - Custom telephony solutions `_ * `ALLOcloud - Telephony in the cloud `_ -* `helpmanual - comprehensive help and man page database `_ -* `bedevere `_ - CPython's GitHub bot, helps maintain and identify issues with a CPython pull request. -* `miss-islington `_ - CPython's GitHub bot, backports and merge CPython's pull requests -* `noa technologies - Bike-sharing management platform `_ - SSE endpoint, pushes real time updates of bikes location. +* `helpmanual - comprehensive help and man page database + `_ +* `bedevere `_ - CPython's GitHub + bot, helps maintain and identify issues with a CPython pull request. +* `miss-islington `_ - + CPython's GitHub bot, backports and merge CPython's pull requests +* `noa technologies - Bike-sharing management platform + `_ - SSE endpoint, pushes real time updates of + bikes location. +* `Wargaming: World of Tanks `_ +* `Yandex `_ +* `Rambler `_ From 36179d178440d1fccf743ca73f051416a6659b78 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Feb 2018 17:17:58 +0300 Subject: [PATCH 06/15] Bump to 3.0.3 --- CHANGES.rst | 7 +++++++ aiohttp/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6cb7a03719e..e76b0ffbd92 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,13 @@ Changelog .. towncrier release notes start +3.0.3 (2018-02-25) +================== + +- Relax ``attrs`` dependency to minimal actually supported version + 17.0.3 The change allows to avoid version conflicts with currently + existing test tools. + 3.0.2 (2018-02-23) ================== diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 9ad7e523876..bdb70f149bc 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.2' +__version__ = '3.0.3' # This relies on each of the submodules having an __all__ variable. diff --git a/setup.py b/setup.py index 82ff5425bff..4220ac931da 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def build_extension(self, ext): raise RuntimeError('Unable to determine version.') -install_requires = ['attrs>=17.4.0', 'chardet>=2.0,<4.0', +install_requires = ['attrs>=17.3.0', 'chardet>=2.0,<4.0', 'multidict>=4.0,<5.0', 'async_timeout>=1.2,<3.0', 'yarl>=1.0,<2.0'] From 903073283fa85bb3e8bb7568d3832a8fd9c06435 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Feb 2018 17:47:15 +0300 Subject: [PATCH 07/15] Make client request .send() a coroutine (#2758) --- aiohttp/client.py | 2 +- aiohttp/client_reqrep.py | 2 +- aiohttp/connector.py | 2 +- tests/test_client_request.py | 100 +++++++++++++++++++---------------- tests/test_proxy.py | 78 +++------------------------ 5 files changed, 64 insertions(+), 120 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 93eff65e604..baa7f0a0564 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -326,7 +326,7 @@ async def _request(self, method, url, *, tcp_nodelay(conn.transport, True) tcp_cork(conn.transport, False) try: - resp = req.send(conn) + resp = await req.send(conn) try: await resp.start(conn, read_until_eof) except BaseException: diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 5c169c07e87..4c3ea6c997d 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -461,7 +461,7 @@ async def write_bytes(self, writer, conn): finally: self._writer = None - def send(self, conn): + async def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 2d9108b0a47..072053b9299 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -893,7 +893,7 @@ async def _create_proxy_connection(self, req, traces=None): proxy_req.url = req.url key = (req.host, req.port, req.ssl) conn = Connection(self, key, proto, self._loop) - proxy_resp = proxy_req.send(conn) + proxy_resp = await proxy_req.send(conn) try: resp = await proxy_resp.start(conn, True) except BaseException: diff --git a/tests/test_client_request.py b/tests/test_client_request.py index f92b4370575..74f9f352dad 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -501,7 +501,7 @@ def test_gen_netloc_no_port(make_request): '012345678901234567890' -def test_connection_header(loop, conn): +async def test_connection_header(loop, conn): req = ClientRequest('get', URL('http://python.org'), loop=loop) req.keep_alive = mock.Mock() req.headers.clear() @@ -509,24 +509,24 @@ def test_connection_header(loop, conn): req.keep_alive.return_value = True req.version = (1, 1) req.headers.clear() - req.send(conn) + await req.send(conn) assert req.headers.get('CONNECTION') is None req.version = (1, 0) req.headers.clear() - req.send(conn) + await req.send(conn) assert req.headers.get('CONNECTION') == 'keep-alive' req.keep_alive.return_value = False req.version = (1, 1) req.headers.clear() - req.send(conn) + await req.send(conn) assert req.headers.get('CONNECTION') == 'close' async def test_no_content_length(loop, conn): req = ClientRequest('get', URL('http://python.org'), loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert req.headers.get('CONTENT-LENGTH') is None await req.close() resp.close() @@ -534,69 +534,69 @@ async def test_no_content_length(loop, conn): async def test_no_content_length_head(loop, conn): req = ClientRequest('head', URL('http://python.org'), loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert req.headers.get('CONTENT-LENGTH') is None await req.close() resp.close() -def test_content_type_auto_header_get(loop, conn): +async def test_content_type_auto_header_get(loop, conn): req = ClientRequest('get', URL('http://python.org'), loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'CONTENT-TYPE' not in req.headers resp.close() -def test_content_type_auto_header_form(loop, conn): +async def test_content_type_auto_header_form(loop, conn): req = ClientRequest('post', URL('http://python.org'), data={'hey': 'you'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'application/x-www-form-urlencoded' == \ req.headers.get('CONTENT-TYPE') resp.close() -def test_content_type_auto_header_bytes(loop, conn): +async def test_content_type_auto_header_bytes(loop, conn): req = ClientRequest('post', URL('http://python.org'), data=b'hey you', loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'application/octet-stream' == req.headers.get('CONTENT-TYPE') resp.close() -def test_content_type_skip_auto_header_bytes(loop, conn): +async def test_content_type_skip_auto_header_bytes(loop, conn): req = ClientRequest('post', URL('http://python.org'), data=b'hey you', skip_auto_headers={'Content-Type'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'CONTENT-TYPE' not in req.headers resp.close() -def test_content_type_skip_auto_header_form(loop, conn): +async def test_content_type_skip_auto_header_form(loop, conn): req = ClientRequest('post', URL('http://python.org'), data={'hey': 'you'}, loop=loop, skip_auto_headers={'Content-Type'}) - resp = req.send(conn) + resp = await req.send(conn) assert 'CONTENT-TYPE' not in req.headers resp.close() -def test_content_type_auto_header_content_length_no_skip(loop, conn): +async def test_content_type_auto_header_content_length_no_skip(loop, conn): req = ClientRequest('post', URL('http://python.org'), data=io.BytesIO(b'hey'), skip_auto_headers={'Content-Length'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert req.headers.get('CONTENT-LENGTH') == '3' resp.close() -def test_urlencoded_formdata_charset(loop, conn): +async def test_urlencoded_formdata_charset(loop, conn): req = ClientRequest( 'post', URL('http://python.org'), data=aiohttp.FormData({'hey': 'you'}, charset='koi8-r'), loop=loop) - req.send(conn) + await req.send(conn) assert 'application/x-www-form-urlencoded; charset=koi8-r' == \ req.headers.get('CONTENT-TYPE') @@ -606,7 +606,7 @@ async def test_post_data(loop, conn): req = ClientRequest( meth, URL('http://python.org/'), data={'life': '42'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert '/' == req.url.path assert b'life=42' == req.body._value assert 'application/x-www-form-urlencoded' ==\ @@ -655,7 +655,7 @@ async def test_bytes_data(loop, conn): req = ClientRequest( meth, URL('http://python.org/'), data=b'binary data', loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert '/' == req.url.path assert isinstance(req.body, payload.BytesPayload) assert b'binary data' == req.body._value @@ -668,7 +668,7 @@ async def test_content_encoding(loop, conn): req = ClientRequest('post', URL('http://python.org/'), data='foo', compress='deflate', loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: - resp = req.send(conn) + resp = await req.send(conn) assert req.headers['TRANSFER-ENCODING'] == 'chunked' assert req.headers['CONTENT-ENCODING'] == 'deflate' m_writer.return_value\ @@ -681,7 +681,7 @@ async def test_content_encoding_dont_set_headers_if_no_body(loop, conn): req = ClientRequest('post', URL('http://python.org/'), compress='deflate', loop=loop) with mock.patch('aiohttp.client_reqrep.http'): - resp = req.send(conn) + resp = await req.send(conn) assert 'TRANSFER-ENCODING' not in req.headers assert 'CONTENT-ENCODING' not in req.headers await req.close() @@ -693,7 +693,7 @@ async def test_content_encoding_header(loop, conn): 'post', URL('http://python.org/'), data='foo', headers={'Content-Encoding': 'deflate'}, loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: - resp = req.send(conn) + resp = await req.send(conn) assert not m_writer.return_value.enable_compression.called assert not m_writer.return_value.enable_chunking.called @@ -712,7 +712,7 @@ async def test_chunked(loop, conn): req = ClientRequest( 'post', URL('http://python.org/'), headers={'TRANSFER-ENCODING': 'gzip'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'gzip' == req.headers['TRANSFER-ENCODING'] await req.close() resp.close() @@ -722,7 +722,7 @@ async def test_chunked2(loop, conn): req = ClientRequest( 'post', URL('http://python.org/'), headers={'Transfer-encoding': 'chunked'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'chunked' == req.headers['TRANSFER-ENCODING'] await req.close() resp.close() @@ -732,7 +732,7 @@ async def test_chunked_explicit(loop, conn): req = ClientRequest( 'post', URL('http://python.org/'), chunked=True, loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: - resp = req.send(conn) + resp = await req.send(conn) assert 'chunked' == req.headers['TRANSFER-ENCODING'] m_writer.return_value.enable_chunking.assert_called_with() @@ -809,20 +809,20 @@ async def test_file_upload_force_chunked(loop): await req.close() -def test_expect100(loop, conn): +async def test_expect100(loop, conn): req = ClientRequest('get', URL('http://python.org/'), expect100=True, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert '100-continue' == req.headers['EXPECT'] assert req._continue is not None req.terminate() resp.close() -def test_expect_100_continue_header(loop, conn): +async def test_expect_100_continue_header(loop, conn): req = ClientRequest('get', URL('http://python.org/'), headers={'expect': '100-continue'}, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert '100-continue' == req.headers['EXPECT'] assert req._continue is not None req.terminate() @@ -840,7 +840,7 @@ def gen(writer): assert req.chunked assert req.headers['TRANSFER-ENCODING'] == 'chunked' - resp = req.send(conn) + resp = await req.send(conn) assert asyncio.isfuture(req._writer) await resp.wait_for_close() assert req._writer is None @@ -858,7 +858,7 @@ async def test_data_file(loop, buf, conn): assert isinstance(req.body, payload.BufferedReaderPayload) assert req.headers['TRANSFER-ENCODING'] == 'chunked' - resp = req.send(conn) + resp = await req.send(conn) assert asyncio.isfuture(req._writer) await resp.wait_for_close() assert req._writer is None @@ -886,7 +886,7 @@ async def throw_exc(): loop.create_task(throw_exc()) - req.send(conn) + await req.send(conn) await req._writer # assert conn.close.called assert conn.protocol.set_exception.called @@ -911,7 +911,7 @@ async def throw_exc(): loop.create_task(throw_exc()) - req.send(conn) + await req.send(conn) await req._writer # assert connection.close.called assert conn.protocol.set_exception.called @@ -940,7 +940,7 @@ async def coro(): loop.create_task(coro()) - resp = req.send(conn) + resp = await req.send(conn) await req._writer assert buf.split(b'\r\n\r\n', 1)[1] == \ b'b\r\nbinary data\r\n7\r\n result\r\n0\r\n\r\n' @@ -959,7 +959,7 @@ async def coro(): loop.create_task(coro()) - resp = req.send(conn) + resp = await req.send(conn) await req._writer assert buf.split(b'\r\n\r\n', 1)[1] == b'data' @@ -975,7 +975,7 @@ async def gen(writer): req = ClientRequest( 'POST', URL('http://python.org/'), data=gen(), loop=loop) - resp = req.send(conn) + resp = await req.send(conn) await req.close() assert buf.split(b'\r\n\r\n', 1)[1] == b'6\r\nresult\r\n0\r\n\r\n' await req.close() @@ -990,7 +990,7 @@ def read(self, decode=False): req = ClientRequest( 'GET', URL('http://python.org/'), response_class=CustomResponse, loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert 'customized!' == resp.read() await req.close() resp.close() @@ -1012,7 +1012,7 @@ async def test_oserror_on_write_bytes(loop, conn): async def test_terminate(loop, conn): req = ClientRequest('get', URL('http://python.org'), loop=loop) - resp = req.send(conn) + resp = await req.send(conn) assert req._writer is not None writer = req._writer = mock.Mock() @@ -1023,12 +1023,18 @@ async def test_terminate(loop, conn): def test_terminate_with_closed_loop(loop, conn): - req = ClientRequest('get', URL('http://python.org'), loop=loop) - resp = req.send(conn) - assert req._writer is not None - writer = req._writer = mock.Mock() + req = resp = writer = None + + async def go(): + nonlocal req, resp, writer + req = ClientRequest('get', URL('http://python.org')) + resp = await req.send(conn) + assert req._writer is not None + writer = req._writer = mock.Mock() + + await asyncio.sleep(0.05) - loop.run_until_complete(asyncio.sleep(0.05, loop=loop)) + loop.run_until_complete(go()) loop.close() req.terminate() @@ -1063,7 +1069,7 @@ async def start(self, connection, read_until_eof=False): class CustomRequest(ClientRequest): - def send(self, conn): + async def send(self, conn): resp = self.response_class(self.method, self.url, writer=self._writer, diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 7ed86e86117..d66e0b7f728 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -93,60 +93,6 @@ def test_proxy_headers(self, ClientRequestMock): loop=self.loop, ssl=None) - @mock.patch('aiohttp.connector.ClientRequest', **clientrequest_mock_attrs) - def test_connect_req_verify_ssl_true(self, ClientRequestMock): - req = ClientRequest( - 'GET', URL('https://www.python.org'), - proxy=URL('http://proxy.example.com'), - loop=self.loop, - ssl=True, - ) - - proto = mock.Mock() - connector = aiohttp.TCPConnector(loop=self.loop) - connector._create_proxy_connection = mock.MagicMock( - side_effect=connector._create_proxy_connection) - connector._create_direct_connection = mock.MagicMock( - side_effect=connector._create_direct_connection) - connector._resolve_host = make_mocked_coro([mock.MagicMock()]) - - self.loop.create_connection = make_mocked_coro( - (proto.transport, proto)) - self.loop.run_until_complete(connector.connect(req)) - - connector._create_proxy_connection.assert_called_with( - req, - traces=None) - ((proxy_req,), _) = connector._create_direct_connection.call_args - proxy_req.send.assert_called_with(mock.ANY) - - @mock.patch('aiohttp.connector.ClientRequest', **clientrequest_mock_attrs) - def test_connect_req_verify_ssl_false(self, ClientRequestMock): - req = ClientRequest( - 'GET', URL('https://www.python.org'), - proxy=URL('http://proxy.example.com'), - loop=self.loop, - ssl=False, - ) - - proto = mock.Mock() - connector = aiohttp.TCPConnector(loop=self.loop) - connector._create_proxy_connection = mock.MagicMock( - side_effect=connector._create_proxy_connection) - connector._create_direct_connection = mock.MagicMock( - side_effect=connector._create_direct_connection) - connector._resolve_host = make_mocked_coro([mock.MagicMock()]) - - self.loop.create_connection = make_mocked_coro( - (proto.transport, proto)) - self.loop.run_until_complete(connector.connect(req)) - - connector._create_proxy_connection.assert_called_with( - req, - traces=None) - ((proxy_req,), _) = connector._create_direct_connection.call_args - proxy_req.send.assert_called_with(mock.ANY) - def test_proxy_auth(self): with self.assertRaises(ValueError) as ctx: ClientRequest( @@ -201,8 +147,7 @@ def test_https_connect(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) @@ -238,8 +183,7 @@ def test_https_connect_certificate_error(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) @@ -281,8 +225,7 @@ def test_https_connect_ssl_error(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) @@ -324,8 +267,7 @@ def test_https_connect_runtime_error(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) @@ -358,8 +300,7 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro( mock.Mock(status=400, reason='bad request')) @@ -393,8 +334,7 @@ def test_https_connect_resp_start_error(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro( raise_exception=OSError("error message")) @@ -461,8 +401,7 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) @@ -506,8 +445,7 @@ def test_https_auth(self, ClientRequestMock): proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop - proxy_req.send = send_mock = mock.Mock() - send_mock.return_value = proxy_resp + proxy_req.send = make_mocked_coro(proxy_resp) proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) connector = aiohttp.TCPConnector(loop=self.loop) From bb31d279075b18d9bb267e172d7686e083f70057 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 25 Feb 2018 23:48:14 +0900 Subject: [PATCH 08/15] Merge pytest.ini into setup.cfg's tool:pytest section (#2757) --- pytest.ini | 4 ---- setup.cfg | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 5dbcbe1fb99..00000000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -addopts= --aiohttp-loop=all -filterwarnings= - error diff --git a/setup.cfg b/setup.cfg index 83339f1b65e..d0a6788fdfc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,8 @@ max-line-length=79 [tool:pytest] testpaths = tests -addopts = --loop all +addopts = --aiohttp-loop=all +filterwarnings = error [isort] known_third_party=jinja2 From 6c92e8f535edc62749cc69d9dbfe6a9826a8dccf Mon Sep 17 00:00:00 2001 From: Terence Honles Date: Sun, 25 Feb 2018 21:06:29 -0800 Subject: [PATCH 09/15] Fix MultipartWriter.append* no longer returning part/payload. (#2759) * fix MultipartWriter.append* no longer returning part/payload - Fixes commit: caa6bdb2484bf821b52a65322efa98a889b593de MultipartWriter.append methods used to return the part appended to the writer so one could set the content_disposition, etc. This patch restores that functionality so the code matches the documentation in multipart.rst - This patch also makes append_json use the JsonPayload object instead of duplicating functionality. * add change entry --- CHANGES/2759.bugfix | 1 + aiohttp/multipart.py | 12 +++++------- docs/multipart.rst | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 CHANGES/2759.bugfix diff --git a/CHANGES/2759.bugfix b/CHANGES/2759.bugfix new file mode 100644 index 00000000000..9deacfbc2ed --- /dev/null +++ b/CHANGES/2759.bugfix @@ -0,0 +1 @@ +Fix MultipartWriter.append* no longer returning part/payload. diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 5e49715bcee..f28a0896ea8 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -14,7 +14,7 @@ CONTENT_TRANSFER_ENCODING, CONTENT_TYPE) from .helpers import CHAR, TOKEN, parse_mimetype, reify from .http import HttpParser -from .payload import (BytesPayload, LookupError, Payload, StringPayload, +from .payload import (JsonPayload, LookupError, Payload, StringPayload, get_payload, payload_type) @@ -712,10 +712,10 @@ def append(self, obj, headers=None): obj.headers.update(headers) else: obj._headers = headers - self.append_payload(obj) + return self.append_payload(obj) else: try: - self.append_payload(get_payload(obj, headers=headers)) + return self.append_payload(get_payload(obj, headers=headers)) except LookupError: raise TypeError @@ -752,16 +752,14 @@ def append_payload(self, payload): ).encode('utf-8') + b'\r\n' self._parts.append((payload, headers, encoding, te_encoding)) + return payload def append_json(self, obj, headers=None): """Helper to append JSON part.""" if headers is None: headers = CIMultiDict() - data = json.dumps(obj).encode('utf-8') - self.append_payload( - BytesPayload( - data, headers=headers, content_type='application/json')) + return self.append_payload(JsonPayload(obj, headers=headers)) def append_form(self, obj, headers=None): """Helper to append form urlencoded part.""" diff --git a/docs/multipart.rst b/docs/multipart.rst index bebdb15159c..2c6cb40483d 100644 --- a/docs/multipart.rst +++ b/docs/multipart.rst @@ -152,9 +152,9 @@ will include the file's basename:: part = root.append(open(__file__, 'rb')) If you want to send a file with a different name, just handle the -:class:`BodyPartWriter` instance which :meth:`MultipartWriter.append` will +:class:`Payload` instance which :meth:`MultipartWriter.append` will always return and set `Content-Disposition` explicitly by using -the :meth:`BodyPartWriter.set_content_disposition` helper:: +the :meth:`Payload.set_content_disposition` helper:: part.set_content_disposition('attachment', filename='secret.txt') From f38e14c790ffb2517419a6520fc8b8d338896952 Mon Sep 17 00:00:00 2001 From: Cyrbuzz Date: Mon, 26 Feb 2018 16:33:27 +0800 Subject: [PATCH 10/15] fix #2760 (#2761) * fix typo * fix typo * fix error. --- docs/client_reference.rst | 14 +++++++------- docs/web_reference.rst | 13 +++++-------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 48ca9bfd9cb..a8e9926b9a9 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -901,7 +901,7 @@ TCPConnector :param bool force_close: close underlying sockets after connection releasing (optional). - :param tuple enable_cleanup_closed: Some ssl servers do not properly complete + :param bool enable_cleanup_closed: Some ssl servers do not properly complete SSL shutdown process, in that case asyncio leaks SSL connections. If this parameter is set to True, aiohttp additionally aborts underlining transport after 2 seconds. It is off by default. @@ -1739,15 +1739,15 @@ Connection errors .. class:: ClientProxyConnectionError - Derived from :exc:`ClientConnectonError` + Derived from :exc:`ClientConnectorError` .. class:: ServerConnectionError - Derived from :exc:`ClientConnectonError` + Derived from :exc:`ClientConnectionError` .. class:: ClientSSLError - Derived from :exc:`ClientConnectonError` + Derived from :exc:`ClientConnectorError` .. class:: ClientConnectorSSLError @@ -1765,7 +1765,7 @@ Connection errors Server disconnected. - Derived from :exc:`ServerDisconnectonError` + Derived from :exc:`ServerDisconnectionError` .. attribute:: message @@ -1776,13 +1776,13 @@ Connection errors Server operation timeout: read timeout, etc. - Derived from :exc:`ServerConnectonError` and :exc:`asyncio.TimeoutError` + Derived from :exc:`ServerConnectionError` and :exc:`asyncio.TimeoutError` .. class:: ServerFingerprintMismatch Server fingerprint mismatch. - Derived from :exc:`ServerConnectonError` + Derived from :exc:`ServerConnectionError` Hierarchy of exceptions diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 87a35416881..99033719abf 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -769,8 +769,8 @@ StreamResponse Response ^^^^^^^^ -.. class:: Response(*, status=200, headers=None, content_type=None, \ - charset=None, body=None, text=None) +.. class:: Response(*, body=None, status=200, reason=None, text=None, \ + headers=None, content_type=None, charset=None) The most usable response class, inherited from :class:`StreamResponse`. @@ -862,7 +862,7 @@ WebSocketResponse operations. Default value is None (no timeout for receive operation) - :param float compress: Enable per-message deflate extension support. + :param bool compress: Enable per-message deflate extension support. False for disabled, default value is True. The class supports ``async for`` statement for iterating over @@ -1761,14 +1761,11 @@ Resource classes hierarchy:: Read-only *name* of resource or ``None``. - .. comethod:: resolve(method, path) + .. comethod:: resolve(request) Resolve resource by finding appropriate :term:`web-handler` for ``(method, path)`` combination. - :param str method: requested HTTP method. - :param str path: *path* part of request. - :return: (*match_info*, *allowed_methods*) pair. *allowed_methods* is a :class:`set` or HTTP methods accepted by @@ -2066,7 +2063,7 @@ The definition is created by functions like :func:`get` or .. function:: route(method, path, handler, *, name=None, expect_handler=None) - Return :class:`RouteDef` for processing ``POST`` requests. See + Return :class:`RouteDef` for processing requests that decided by ``method``. See :meth:`UrlDispatcher.add_route` for information about parameters. .. versionadded:: 2.3 From 0a94431b813485729eea71d2ff0f633518de6484 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Feb 2018 13:34:37 +0300 Subject: [PATCH 11/15] Async write headers (#2762) * Make _start a coroutine * Make write_headers a coroutine * Add change note --- CHANGES/2762.feature | 1 + aiohttp/client_reqrep.py | 2 +- aiohttp/http_writer.py | 2 +- aiohttp/test_utils.py | 13 +++---- aiohttp/web_response.py | 32 ++++++++-------- tests/test_client_request.py | 4 ++ tests/test_web_exceptions.py | 4 +- tests/test_web_response.py | 74 +++++++++++++++++++----------------- tests/test_web_websocket.py | 16 ++------ 9 files changed, 72 insertions(+), 76 deletions(-) create mode 100644 CHANGES/2762.feature diff --git a/CHANGES/2762.feature b/CHANGES/2762.feature new file mode 100644 index 00000000000..add0440fc7e --- /dev/null +++ b/CHANGES/2762.feature @@ -0,0 +1 @@ +Make ``writer.write_headers`` a coroutine. \ No newline at end of file diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 4c3ea6c997d..4c7b8dca01c 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -505,7 +505,7 @@ async def send(self, conn): # status + headers status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, path, self.version) - writer.write_headers(status_line, self.headers) + await writer.write_headers(status_line, self.headers) self._writer = self.loop.create_task(self.write_bytes(writer, conn)) diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 9e25ddc81b8..83212ca9a88 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -91,7 +91,7 @@ def write(self, chunk, *, drain=True, LIMIT=64*1024): return noop() - def write_headers(self, status_line, headers, SEP=': ', END='\r\n'): + async def write_headers(self, status_line, headers, SEP=': ', END='\r\n'): """Write request/response status and headers.""" # status + headers headers = status_line + ''.join( diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 3804faeb211..0253daf80e7 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -463,7 +463,6 @@ def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, - payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, @@ -509,14 +508,12 @@ def make_mocked_request(method, path, headers=None, *, if writer is sentinel: writer = mock.Mock() + writer.write_headers = make_mocked_coro(None) + writer.write = make_mocked_coro(None) + writer.write_eof = make_mocked_coro(None) + writer.drain = make_mocked_coro(None) writer.transport = transport - if payload_writer is sentinel: - payload_writer = mock.Mock() - payload_writer.write = make_mocked_coro(None) - payload_writer.write_eof = make_mocked_coro(None) - payload_writer.drain = make_mocked_coro(None) - protocol.transport = transport protocol.writer = writer @@ -524,7 +521,7 @@ def make_mocked_request(method, path, headers=None, *, payload = mock.Mock() req = Request(message, payload, - protocol, payload_writer, task, loop, + protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index e5fd254849e..fb07712965a 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -297,19 +297,19 @@ async def prepare(self, request): return self._payload_writer await request._prepare_hook(self) - return self._start(request) - - def _start(self, request, - HttpVersion10=HttpVersion10, - HttpVersion11=HttpVersion11, - CONNECTION=hdrs.CONNECTION, - DATE=hdrs.DATE, - SERVER=hdrs.SERVER, - CONTENT_TYPE=hdrs.CONTENT_TYPE, - CONTENT_LENGTH=hdrs.CONTENT_LENGTH, - SET_COOKIE=hdrs.SET_COOKIE, - SERVER_SOFTWARE=SERVER_SOFTWARE, - TRANSFER_ENCODING=hdrs.TRANSFER_ENCODING): + return await self._start(request) + + async def _start(self, request, + HttpVersion10=HttpVersion10, + HttpVersion11=HttpVersion11, + CONNECTION=hdrs.CONNECTION, + DATE=hdrs.DATE, + SERVER=hdrs.SERVER, + CONTENT_TYPE=hdrs.CONTENT_TYPE, + CONTENT_LENGTH=hdrs.CONTENT_LENGTH, + SET_COOKIE=hdrs.SET_COOKIE, + SERVER_SOFTWARE=SERVER_SOFTWARE, + TRANSFER_ENCODING=hdrs.TRANSFER_ENCODING): self._req = request keep_alive = self._keep_alive @@ -364,7 +364,7 @@ def _start(self, request, # status line status_line = 'HTTP/{}.{} {} {}\r\n'.format( version[0], version[1], self._status, self._reason) - writer.write_headers(status_line, headers) + await writer.write_headers(status_line, headers) return writer @@ -594,7 +594,7 @@ async def write_eof(self): else: await super().write_eof() - def _start(self, request): + async def _start(self, request): if not self._chunked and hdrs.CONTENT_LENGTH not in self._headers: if not self._body_payload: if self._body is not None: @@ -602,7 +602,7 @@ def _start(self, request): else: self._headers[hdrs.CONTENT_LENGTH] = '0' - return super()._start(request) + return await super()._start(request) def _do_start_compression(self, coding): if self._body_payload or self._chunked: diff --git a/tests/test_client_request.py b/tests/test_client_request.py index 74f9f352dad..bbabf2d463a 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -17,6 +17,7 @@ from aiohttp import BaseConnector, hdrs, payload from aiohttp.client_reqrep import (ClientRequest, ClientResponse, Fingerprint, _merge_ssl_params) +from aiohttp.test_utils import make_mocked_coro @pytest.fixture @@ -668,6 +669,7 @@ async def test_content_encoding(loop, conn): req = ClientRequest('post', URL('http://python.org/'), data='foo', compress='deflate', loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: + m_writer.return_value.write_headers = make_mocked_coro() resp = await req.send(conn) assert req.headers['TRANSFER-ENCODING'] == 'chunked' assert req.headers['CONTENT-ENCODING'] == 'deflate' @@ -693,6 +695,7 @@ async def test_content_encoding_header(loop, conn): 'post', URL('http://python.org/'), data='foo', headers={'Content-Encoding': 'deflate'}, loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: + m_writer.return_value.write_headers = make_mocked_coro() resp = await req.send(conn) assert not m_writer.return_value.enable_compression.called @@ -732,6 +735,7 @@ async def test_chunked_explicit(loop, conn): req = ClientRequest( 'post', URL('http://python.org/'), chunked=True, loop=loop) with mock.patch('aiohttp.client_reqrep.StreamWriter') as m_writer: + m_writer.return_value.write_headers = make_mocked_coro() resp = await req.send(conn) assert 'chunked' == req.headers['TRANSFER-ENCODING'] diff --git a/tests/test_web_exceptions.py b/tests/test_web_exceptions.py index 0c280bd5eef..93c9cd76f60 100644 --- a/tests/test_web_exceptions.py +++ b/tests/test_web_exceptions.py @@ -24,7 +24,7 @@ def append(data=b''): buf.extend(data) return helpers.noop() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): headers = status_line + ''.join( [k + ': ' + v + '\r\n' for k, v in headers.items()]) headers = headers.encode('utf-8') + b'\r\n' @@ -39,7 +39,7 @@ def write_headers(status_line, headers): app._debug = False app.on_response_prepare = signals.Signal(app) app.on_response_prepare.freeze() - req = make_mocked_request(method, path, app=app, payload_writer=writer) + req = make_mocked_request(method, path, app=app, writer=writer) return req diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 987cdb20da6..ba2f8d1bd09 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -45,7 +45,7 @@ def buffer_data(chunk): def write(chunk): buf.extend(chunk) - def write_headers(status_line, headers): + async def write_headers(status_line, headers): headers = status_line + ''.join( [k + ': ' + v + '\r\n' for k, v in headers.items()]) headers = headers.encode('utf-8') + b'\r\n' @@ -235,7 +235,7 @@ def test_last_modified_reset(): async def test_start(): - req = make_request('GET', '/', payload_writer=mock.Mock()) + req = make_request('GET', '/') resp = StreamResponse() assert resp.keep_alive is None @@ -274,7 +274,7 @@ def test_enable_chunked_encoding_with_content_length(): async def test_chunk_size(): - req = make_request('GET', '/', payload_writer=mock.Mock()) + req = make_request('GET', '/') resp = StreamResponse() assert not resp.chunked @@ -300,7 +300,7 @@ async def test_chunked_encoding_forbidden_for_http_10(): async def test_compression_no_accept(): - req = make_request('GET', '/', payload_writer=mock.Mock()) + req = make_request('GET', '/') resp = StreamResponse() assert not resp.chunked @@ -313,7 +313,7 @@ async def test_compression_no_accept(): async def test_force_compression_no_accept_backwards_compat(): - req = make_request('GET', '/', payload_writer=mock.Mock()) + req = make_request('GET', '/') resp = StreamResponse() assert not resp.chunked @@ -327,7 +327,7 @@ async def test_force_compression_no_accept_backwards_compat(): async def test_force_compression_false_backwards_compat(): - req = make_request('GET', '/', payload_writer=mock.Mock()) + req = make_request('GET', '/') resp = StreamResponse() assert not resp.compression @@ -421,13 +421,13 @@ async def test_change_content_length_if_compression_enabled(): async def test_set_content_length_if_compression_enabled(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH in headers assert headers[hdrs.CONTENT_LENGTH] == '26' assert hdrs.TRANSFER_ENCODING not in headers writer.write_headers.side_effect = write_headers - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) resp = Response(body=b'answer') resp.enable_compression(ContentCoding.gzip) @@ -440,12 +440,12 @@ def write_headers(status_line, headers): async def test_remove_content_length_if_compression_enabled_http11(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH not in headers assert headers.get(hdrs.TRANSFER_ENCODING, '') == 'chunked' writer.write_headers.side_effect = write_headers - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) resp = StreamResponse() resp.content_length = 123 resp.enable_compression(ContentCoding.gzip) @@ -456,13 +456,13 @@ def write_headers(status_line, headers): async def test_remove_content_length_if_compression_enabled_http10(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH not in headers assert hdrs.TRANSFER_ENCODING not in headers writer.write_headers.side_effect = write_headers req = make_request('GET', '/', version=HttpVersion10, - payload_writer=writer) + writer=writer) resp = StreamResponse() resp.content_length = 123 resp.enable_compression(ContentCoding.gzip) @@ -473,13 +473,13 @@ def write_headers(status_line, headers): async def test_force_compression_identity(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH in headers assert hdrs.TRANSFER_ENCODING not in headers writer.write_headers.side_effect = write_headers req = make_request('GET', '/', - payload_writer=writer) + writer=writer) resp = StreamResponse() resp.content_length = 123 resp.enable_compression(ContentCoding.identity) @@ -490,28 +490,28 @@ def write_headers(status_line, headers): async def test_force_compression_identity_response(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert headers[hdrs.CONTENT_LENGTH] == "6" assert hdrs.TRANSFER_ENCODING not in headers writer.write_headers.side_effect = write_headers req = make_request('GET', '/', - payload_writer=writer) + writer=writer) resp = Response(body=b'answer') resp.enable_compression(ContentCoding.identity) await resp.prepare(req) assert resp.content_length == 6 -async def test_remove_content_length_if_compression_enabled_on_payload_http11(): # noqa +async def test_rm_content_length_if_compression_enabled_on_payload_http11(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH not in headers assert headers.get(hdrs.TRANSFER_ENCODING, '') == 'chunked' writer.write_headers.side_effect = write_headers - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) payload = BytesPayload(b'answer', headers={"X-Test-Header": "test"}) resp = Response(body=payload) assert resp.content_length == 6 @@ -521,16 +521,16 @@ def write_headers(status_line, headers): assert resp.content_length is None -async def test_remove_content_length_if_compression_enabled_on_payload_http10(): # noqa +async def test_rm_content_length_if_compression_enabled_on_payload_http10(): writer = mock.Mock() - def write_headers(status_line, headers): + async def write_headers(status_line, headers): assert hdrs.CONTENT_LENGTH not in headers assert hdrs.TRANSFER_ENCODING not in headers writer.write_headers.side_effect = write_headers req = make_request('GET', '/', version=HttpVersion10, - payload_writer=writer) + writer=writer) resp = Response(body=BytesPayload(b'answer')) resp.enable_compression(ContentCoding.gzip) await resp.prepare(req) @@ -788,7 +788,7 @@ def test_response_ctor(): assert 'CONTENT-LENGTH' not in resp.headers -def test_ctor_with_headers_and_status(): +async def test_ctor_with_headers_and_status(): resp = Response(body=b'body', status=201, headers={'Age': '12', 'DATE': 'date'}) @@ -796,7 +796,8 @@ def test_ctor_with_headers_and_status(): assert b'body' == resp.body assert resp.headers['AGE'] == '12' - resp._start(mock.Mock(version=HttpVersion11)) + req = make_mocked_request('GET', '/') + await resp._start(req) assert 4 == resp.content_length assert resp.headers['CONTENT-LENGTH'] == '4' @@ -816,7 +817,7 @@ def test_ctor_text_body_combined(): Response(body=b'123', text='test text') -def test_ctor_text(): +async def test_ctor_text(): resp = Response(text='test text') assert 200 == resp.status @@ -829,7 +830,8 @@ def test_ctor_text(): assert resp.text == 'test text' resp.headers['DATE'] = 'date' - resp._start(mock.Mock(version=HttpVersion11)) + req = make_mocked_request('GET', '/', version=HttpVersion11) + await resp._start(req) assert resp.headers['CONTENT-LENGTH'] == '9' @@ -889,7 +891,7 @@ def test_ctor_both_charset_param_and_header(): charset='koi8-r') -def test_assign_nonbyteish_body(): +async def test_assign_nonbyteish_body(): resp = Response(body=b'data') with pytest.raises(ValueError): @@ -898,7 +900,8 @@ def test_assign_nonbyteish_body(): assert 4 == resp.content_length resp.headers['DATE'] = 'date' - resp._start(mock.Mock(version=HttpVersion11)) + req = make_mocked_request('GET', '/', version=HttpVersion11) + await resp._start(req) assert resp.headers['CONTENT-LENGTH'] == '4' assert 4 == resp.content_length @@ -919,7 +922,7 @@ def test_response_set_content_length(): async def test_send_headers_for_empty_body(buf, writer): - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) resp = Response() await resp.prepare(req) @@ -933,7 +936,7 @@ async def test_send_headers_for_empty_body(buf, writer): async def test_render_with_body(buf, writer): - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) resp = Response(body=b'data') await resp.prepare(req) @@ -951,7 +954,7 @@ async def test_render_with_body(buf, writer): async def test_send_set_cookie_header(buf, writer): resp = Response() resp.cookies['name'] = 'value' - req = make_request('GET', '/', payload_writer=writer) + req = make_request('GET', '/', writer=writer) await resp.prepare(req) await resp.write_eof() @@ -966,16 +969,17 @@ async def test_send_set_cookie_header(buf, writer): async def test_consecutive_write_eof(): - payload_writer = mock.Mock() - payload_writer.write_eof = make_mocked_coro() - req = make_request('GET', '/', payload_writer=payload_writer) + writer = mock.Mock() + writer.write_eof = make_mocked_coro() + writer.write_headers = make_mocked_coro() + req = make_request('GET', '/', writer=writer) data = b'data' resp = Response(body=data) await resp.prepare(req) await resp.write_eof() await resp.write_eof() - payload_writer.write_eof.assert_called_once_with(data) + writer.write_eof.assert_called_once_with(data) def test_set_text_with_content_type(): diff --git a/tests/test_web_websocket.py b/tests/test_web_websocket.py index ef91ef7f09a..649ac66e4b4 100644 --- a/tests/test_web_websocket.py +++ b/tests/test_web_websocket.py @@ -21,16 +21,6 @@ def app(loop): return ret -@pytest.fixture -def writer(loop): - writer = mock.Mock() - writer.drain.return_value = loop.create_future() - writer.drain.return_value.set_result(None) - writer.write_eof.return_value = loop.create_future() - writer.write_eof.return_value.set_result(None) - return writer - - @pytest.fixture def protocol(): ret = mock.Mock() @@ -39,7 +29,7 @@ def protocol(): @pytest.fixture -def make_request(app, protocol, writer): +def make_request(app, protocol): def maker(method, path, headers=None, protocols=False): if headers is None: headers = CIMultiDict( @@ -54,7 +44,7 @@ def maker(method, path, headers=None, protocols=False): return make_mocked_request( method, path, headers, - app=app, protocol=protocol, payload_writer=writer, + app=app, protocol=protocol, loop=app.loop) return maker @@ -301,7 +291,7 @@ async def test_pong_closed(make_request, mocker): assert ws_logger.warning.called -async def test_close_idempotent(make_request, writer): +async def test_close_idempotent(make_request): req = make_request('GET', '/') ws = WebSocketResponse() await ws.prepare(req) From 879f50b0af6670140ed6a76acaf66b9951fe21ab Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Feb 2018 15:18:51 +0300 Subject: [PATCH 12/15] Fix #2752: Get rid of IndexError in aiohttp web server (#2763) --- CHANGES/2752.bugfix | 1 + aiohttp/web_protocol.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 CHANGES/2752.bugfix diff --git a/CHANGES/2752.bugfix b/CHANGES/2752.bugfix new file mode 100644 index 00000000000..b53fe016fd1 --- /dev/null +++ b/CHANGES/2752.bugfix @@ -0,0 +1 @@ +Fix ``IndexError`` in HTTP request handling by server. \ No newline at end of file diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 8b52de17ffb..0ecf4542bbd 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -244,12 +244,14 @@ def data_received(self, data): 500, exc)) self.close() else: - for (msg, payload) in messages: - self._request_count += 1 - self._messages.append((msg, payload)) - - if self._waiter: - self._waiter.set_result(None) + if messages: + # sometimes the parser returns no messages + for (msg, payload) in messages: + self._request_count += 1 + self._messages.append((msg, payload)) + + if self._waiter is not None: + self._waiter.set_result(None) self._upgraded = upgraded if upgraded and tail: From f41907de3519c156be257bc5964026a81069958a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Feb 2018 15:35:53 +0300 Subject: [PATCH 13/15] Relax pypy to latest pypy3.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 285f7584d19..9802e8af1be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - &mainstream_python 3.6 - 3.6-dev - nightly -- &pypy3 pypy3.5-5.8.0 +- &pypy3 pypy3.5 install: - &upgrade_python_toolset pip install --upgrade pip wheel setuptools From 5d5e9fd78034cabddc6b392fb454367f5d91f4bf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Feb 2018 15:37:13 +0300 Subject: [PATCH 14/15] Bump to 3.0.4 --- CHANGES.rst | 7 +++++++ CHANGES/2752.bugfix | 1 - CHANGES/2759.bugfix | 1 - aiohttp/__init__.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 CHANGES/2752.bugfix delete mode 100644 CHANGES/2759.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index e76b0ffbd92..31aa2f2e2ef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,13 @@ Changelog .. towncrier release notes start +3.0.4 (2018-02-26) +================== + +- Fix ``IndexError`` in HTTP request handling by server. (#2752) +- Fix MultipartWriter.append* no longer returning part/payload. (#2759) + + 3.0.3 (2018-02-25) ================== diff --git a/CHANGES/2752.bugfix b/CHANGES/2752.bugfix deleted file mode 100644 index b53fe016fd1..00000000000 --- a/CHANGES/2752.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix ``IndexError`` in HTTP request handling by server. \ No newline at end of file diff --git a/CHANGES/2759.bugfix b/CHANGES/2759.bugfix deleted file mode 100644 index 9deacfbc2ed..00000000000 --- a/CHANGES/2759.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix MultipartWriter.append* no longer returning part/payload. diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index bdb70f149bc..6c6ef8235e0 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.3' +__version__ = '3.0.4' # This relies on each of the submodules having an __all__ variable. From ef560f5649d8c8caae9c6a01b660b44cab8d4922 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 26 Feb 2018 14:32:29 +0100 Subject: [PATCH 15/15] Scheduled weekly dependency update for week 08 (#2764) * Update pytest from 3.4.0 to 3.4.1 * Update pytest from 3.4.0 to 3.4.1 * Update pytest-xdist from 1.22.0 to 1.22.1 * Update sphinx from 1.7.0 to 1.7.1 * Update sphinxcontrib-spelling from 4.0.1 to 4.1.0 --- requirements/ci-wheel.txt | 4 ++-- requirements/doc-spelling.txt | 2 +- requirements/doc.txt | 2 +- requirements/wheel.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/ci-wheel.txt b/requirements/ci-wheel.txt index af19d9e1e99..6b5527f8397 100644 --- a/requirements/ci-wheel.txt +++ b/requirements/ci-wheel.txt @@ -11,10 +11,10 @@ isort==4.3.4 pip==9.0.1 pyflakes==1.6.0 multidict==4.1.0 -pytest==3.4.0 +pytest==3.4.1 pytest-cov==2.5.1 pytest-mock==1.7.0 -pytest-xdist==1.22.0 +pytest-xdist==1.22.1 towncrier==17.8.0 tox==2.9.1 twine==1.9.1 diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt index 81a26ccf618..5c334b0ccc1 100644 --- a/requirements/doc-spelling.txt +++ b/requirements/doc-spelling.txt @@ -1 +1 @@ -sphinxcontrib-spelling==4.0.1; platform_system!="Windows" # We only use it in Travis CI +sphinxcontrib-spelling==4.1.0; platform_system!="Windows" # We only use it in Travis CI diff --git a/requirements/doc.txt b/requirements/doc.txt index 700a3be7fc3..851c9a1708e 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,4 +1,4 @@ -sphinx==1.7.0 +sphinx==1.7.1 sphinxcontrib-asyncio==0.2.0 pygments>=2.1 aiohttp-theme==0.1.4 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index f58a47fc54b..64b26329f26 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ cython==0.27.3 -pytest==3.4.0 +pytest==3.4.1 twine==1.9.1