From 3e62183f93248d11a1f721760800f750f1980e6f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Jun 2017 14:42:39 +0300 Subject: [PATCH 001/167] Bump to 2.3.0a0 --- CHANGES.rst | 27 +++++++++++++++++++++++++++ aiohttp/__init__.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 97a027979aa..3a89d816539 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,33 @@ Changes ======= +2.3.0 (2017-xx-xx) +------------------ + +- + +- + +- + +- + +- + +- + +- + +- + +- + +- + +- + +- + 2.2.0 (2017-06-20) ------------------ diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 3443e79e349..6b89bc0db87 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.2.0' +__version__ = '2.3.0a0' # This relies on each of the submodules having an __all__ variable. From 76ba6c3dd643653dcead687fc876337d76153eae Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Jun 2017 15:01:57 +0300 Subject: [PATCH 002/167] Fix #1562: Add trafaret_config in requirements --- demos/polls/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/demos/polls/requirements.txt b/demos/polls/requirements.txt index 1cc98029514..8e7ece2e35b 100644 --- a/demos/polls/requirements.txt +++ b/demos/polls/requirements.txt @@ -1,3 +1,4 @@ -e . docker-py==1.10.6 pytest-aiohttp==0.1.3 +trafaret_config==1.0.1 From f37a2000ccfa1bd399eaf1fee0aa4f9bc6481fc2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Jun 2017 16:40:24 +0300 Subject: [PATCH 003/167] Fix #1990: drop loop mentioning in docs --- docs/client.rst | 2 +- docs/client_reference.rst | 25 +++++++++++++++---------- docs/web_reference.rst | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index d02fbfcc064..c99c1cb4e34 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -730,7 +730,7 @@ reading procedures:: import async_timeout - with async_timeout.timeout(0.001, loop=session.loop): + with async_timeout.timeout(0.001): async with session.get('https://github.com') as r: await r.text() diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 4244e300998..37478d67163 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -28,8 +28,8 @@ Usage example:: assert resp.status == 200 return await resp.text() - async def main(loop): - async with aiohttp.ClientSession(loop=loop) as client: + async def main(): + async with aiohttp.ClientSession() as client: html = await fetch(client) print(html) @@ -45,7 +45,8 @@ The client session supports the context manager protocol for self closing. headers=None, skip_auto_headers=None, \ auth=None, json_serialize=func:`json.dumps`, \ version=aiohttp.HttpVersion11, \ - cookie_jar=None, read_timeout=None, conn_timeout=None, \ + cookie_jar=None, read_timeout=None, \ + conn_timeout=None, \ raise_for_status=False) The class for creating client sessions and making requests. @@ -63,6 +64,8 @@ The client session supports the context manager protocol for self closing. :func:`asyncio.get_event_loop` is used for getting default event loop otherwise. + .. deprecated:: 2.0 + :param dict cookies: Cookies to send with the request (optional) :param headers: HTTP Headers to send with every request (optional). @@ -500,7 +503,8 @@ keepaliving, cookies and complex connection stuff like properly configured SSL certification chaining. -.. coroutinefunction:: request(method, url, *, params=None, data=None, json=None,\ +.. coroutinefunction:: request(method, url, *, params=None, data=None, \ + json=None,\ headers=None, cookies=None, auth=None, \ allow_redirects=True, max_redirects=10, \ encoding='utf-8', \ @@ -558,10 +562,9 @@ certification chaining. :param loop: :ref:`event loop` used for processing HTTP requests. If param is ``None``, :func:`asyncio.get_event_loop` - is used for getting default event loop, but we strongly - recommend to use explicit loops everywhere. - (optional) + is used for getting default event loop. + .. deprecated:: 2.0 :return ClientResponse: a :class:`client response ` object. @@ -628,9 +631,9 @@ BaseConnector :param loop: :ref:`event loop` used for handling connections. If param is ``None``, :func:`asyncio.get_event_loop` - is used for getting default event loop, but we strongly - recommend to use explicit loops everywhere. - (optional) + is used for getting default event loop. + + .. deprecated:: 2.0 .. attribute:: closed @@ -1302,6 +1305,8 @@ CookieJar :param bool loop: an :ref:`event loop` instance. See :class:`aiohttp.abc.AbstractCookieJar` + .. deprecated:: 2.0 + .. method:: update_cookies(cookies, response_url=None) Update cookies returned by server in ``Set-Cookie`` header. diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 53992f0567f..48185cf130c 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1174,7 +1174,12 @@ duplicated like one using :meth:`Application.copy`. :param debug: Switches debug mode. - :param loop: loop parameter is deprecated. loop is get set during freeze stage. + :param loop: event loop + + .. deprecated:: 2.0 + + The parameter is deprecated. Loop is get set during freeze + stage. .. attribute:: router @@ -1264,6 +1269,8 @@ duplicated like one using :meth:`Application.copy`. If param is ``None`` :func:`asyncio.get_event_loop` used for getting default event loop. + .. deprecated:: 2.0 + :param tuple secure_proxy_ssl_header: Default: ``None``. .. deprecated:: 2.1 @@ -1309,7 +1316,7 @@ duplicated like one using :meth:`Application.copy`. loop = asyncio.get_event_loop() - app = Application(loop=loop) + app = Application() # setup route table # app.router.add_route(...) @@ -2193,7 +2200,8 @@ Middlewares Normalize path middleware ^^^^^^^^^^^^^^^^^^^^^^^^^ -.. function:: normalize_path_middleware(*, append_slash=True, merge_slashes=True) +.. function:: normalize_path_middleware(*, \ + append_slash=True, merge_slashes=True) Middleware that normalizes the path of a request. By normalizing it means: @@ -2206,11 +2214,11 @@ Normalize path middleware and 3) both merge_slashes and append_slash. If the path resolves with at least one of those conditions, it will redirect to the new path. - If append_slash is True append slash when needed. If a resource is + If *append_slash* is True append slash when needed. If a resource is defined with trailing slash and the request comes without it, it will append it automatically. - If merge_slashes is True, merge multiple consecutive slashes in the + If *merge_slashes* is True, merge multiple consecutive slashes in the path into one. From 44ebcacd9f9827f6d6f86866d8be122a7bfb7be3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 20 Jun 2017 18:39:59 +0300 Subject: [PATCH 004/167] make runs make test by default --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 45746b38383..1c0f61933c0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ # Some simple testing tasks (sorry, UNIX only). +all: test + .install-deps: requirements-dev.txt @pip install -U -r requirements-dev.txt @touch .install-deps From 6f2adacff9b37c90aea4e416d73c7f2305d279ad Mon Sep 17 00:00:00 2001 From: Vladyslav Bondar Date: Wed, 21 Jun 2017 12:23:17 +0300 Subject: [PATCH 005/167] Improvement for normalize_path_middleware (#1995) * Improvement for normalize_path_middleware. Now it could handle URLs with query string * Pull request number. --- CHANGES.rst | 3 ++- CONTRIBUTORS.txt | 1 + aiohttp/web_middlewares.py | 11 +++++++++-- tests/test_web_middleware.py | 37 ++++++++++++++++++++++++++++++++---- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3a89d816539..e9ac893d0aa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changes 2.3.0 (2017-xx-xx) ------------------ -- +- Improvement for `normalize_path_middleware`. Added possibility to handle + URLs with query string. #1995 - diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index c20e0cb217f..3d81394e374 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -173,6 +173,7 @@ Vladimir Kozlovski Vladimir Rutsky Vladimir Shulyak Vladimir Zakharov +Vladyslav Bondar W. Trevor King Will McGugan Willem de Groot diff --git a/aiohttp/web_middlewares.py b/aiohttp/web_middlewares.py index 8d17afb30f8..d0d0503823b 100644 --- a/aiohttp/web_middlewares.py +++ b/aiohttp/web_middlewares.py @@ -54,7 +54,14 @@ def middleware(request): if isinstance(request.match_info.route, SystemRoute): paths_to_check = [] - path = request.raw_path + if '?' in request.raw_path: + path, query = request.raw_path.split('?', 1) + if query: + query = '?' + query + else: + query = '' + path = request.raw_path + if merge_slashes: paths_to_check.append(re.sub('//+', '/', path)) if append_slash and not request.path.endswith('/'): @@ -67,7 +74,7 @@ def middleware(request): resolves, request = yield from _check_request_resolves( request, path) if resolves: - return redirect_class(request.path) + return redirect_class(request.path + query) return (yield from handler(request)) diff --git a/tests/test_web_middleware.py b/tests/test_web_middleware.py index 11879702c65..ed36934583c 100644 --- a/tests/test_web_middleware.py +++ b/tests/test_web_middleware.py @@ -118,7 +118,11 @@ class TestNormalizePathMiddleware: ('/resource1', 200), ('/resource1/', 404), ('/resource2', 200), - ('/resource2/', 200) + ('/resource2/', 200), + ('/resource1?p1=1&p2=2', 200), + ('/resource1/?p1=1&p2=2', 404), + ('/resource2?p1=1&p2=2', 200), + ('/resource2/?p1=1&p2=2', 200) ]) def test_add_trailing_when_necessary( self, path, status, cli): @@ -134,7 +138,11 @@ def test_add_trailing_when_necessary( ('/resource1', 200), ('/resource1/', 404), ('/resource2', 404), - ('/resource2/', 200) + ('/resource2/', 200), + ('/resource1?p1=1&p2=2', 200), + ('/resource1/?p1=1&p2=2', 404), + ('/resource2?p1=1&p2=2', 404), + ('/resource2/?p1=1&p2=2', 200) ]) def test_no_trailing_slash_when_disabled( self, path, status, cli): @@ -153,7 +161,13 @@ def test_no_trailing_slash_when_disabled( ('//resource1//a//b/', 404), ('///resource1//a//b', 200), ('/////resource1/a///b', 200), - ('/////resource1/a//b/', 404) + ('/////resource1/a//b/', 404), + ('/resource1/a/b?p=1', 200), + ('//resource1//a//b?p=1', 200), + ('//resource1//a//b/?p=1', 404), + ('///resource1//a//b?p=1', 200), + ('/////resource1/a///b?p=1', 200), + ('/////resource1/a//b/?p=1', 404), ]) def test_merge_slash(self, path, status, cli): extra_middlewares = [ @@ -179,7 +193,22 @@ def test_merge_slash(self, path, status, cli): ('///resource2//a//b', 200), ('///resource2//a//b/', 200), ('/////resource2/a///b', 200), - ('/////resource2/a///b/', 200) + ('/////resource2/a///b/', 200), + ('/resource1/a/b?p=1', 200), + ('/resource1/a/b/?p=1', 404), + ('//resource2//a//b?p=1', 200), + ('//resource2//a//b/?p=1', 200), + ('///resource1//a//b?p=1', 200), + ('///resource1//a//b/?p=1', 404), + ('/////resource1/a///b?p=1', 200), + ('/////resource1/a///b/?p=1', 404), + ('/resource2/a/b?p=1', 200), + ('//resource2//a//b?p=1', 200), + ('//resource2//a//b/?p=1', 200), + ('///resource2//a//b?p=1', 200), + ('///resource2//a//b/?p=1', 200), + ('/////resource2/a///b?p=1', 200), + ('/////resource2/a///b/?p=1', 200) ]) def test_append_and_merge_slash(self, path, status, cli): extra_middlewares = [ From 075c34c8f98d3690f7167d8f885bfbd3d74dd67a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 22 Jun 2017 20:28:24 +0300 Subject: [PATCH 006/167] Adopt to multidict 3.0.0 (#2000) * Fix couple tests * Another test * Fix last tests * Pin multidict requirement to 3.0.0 * Fix tests * Make test stable --- aiohttp/client_reqrep.py | 2 +- requirements-ci.txt | 2 +- tests/test_client_session.py | 17 +++++++---------- tests/test_http_parser.py | 6 +++--- tests/test_multipart.py | 2 +- tests/test_web_request.py | 4 ++-- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index b1a9939aec7..ee810cf4564 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -161,7 +161,7 @@ def update_headers(self, headers): def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = CIMultiDict( - (hdr, None) for hdr in skip_auto_headers) + (hdr, None) for hdr in sorted(skip_auto_headers)) used_headers = self.headers.copy() used_headers.extend(self.skip_auto_headers) diff --git a/requirements-ci.txt b/requirements-ci.txt index e40e3143441..f7f27755934 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -8,7 +8,7 @@ cython==0.25.2 chardet==3.0.4 isort==4.2.15 tox==2.7.0 -multidict==2.1.6 +multidict==3.0.0 async-timeout==1.2.1 sphinxcontrib-asyncio==0.2.0 sphinxcontrib-newsfeed==0.1.4 diff --git a/tests/test_client_session.py b/tests/test_client_session.py index cd7b8adab6c..fc01f5198e4 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -67,7 +67,7 @@ def test_init_headers_simple_dict(create_session): session = create_session(headers={"h1": "header1", "h2": "header2"}) assert (sorted(session._default_headers.items()) == - ([("H1", "header1"), ("H2", "header2")])) + ([("h1", "header1"), ("h2", "header2")])) def test_init_headers_list_of_tuples(create_session): @@ -126,8 +126,7 @@ def test_merge_headers(create_session): headers = session._prepare_headers({"h1": "h1"}) assert isinstance(headers, CIMultiDict) - assert headers == CIMultiDict([("h2", "header2"), - ("h1", "h1")]) + assert headers == {"h1": "h1", "h2": "header2"} def test_merge_headers_with_multi_dict(create_session): @@ -135,8 +134,7 @@ def test_merge_headers_with_multi_dict(create_session): "h2": "header2"}) headers = session._prepare_headers(MultiDict([("h1", "h1")])) assert isinstance(headers, CIMultiDict) - assert headers == CIMultiDict([("h2", "header2"), - ("h1", "h1")]) + assert headers == {"h1": "h1", "h2": "header2"} def test_merge_headers_with_list_of_tuples(create_session): @@ -144,8 +142,7 @@ def test_merge_headers_with_list_of_tuples(create_session): "h2": "header2"}) headers = session._prepare_headers([("h1", "h1")]) assert isinstance(headers, CIMultiDict) - assert headers == CIMultiDict([("h2", "header2"), - ("h1", "h1")]) + assert headers == {"h1": "h1", "h2": "header2"} def test_merge_headers_with_list_of_tuples_duplicated_names(create_session): @@ -156,9 +153,9 @@ def test_merge_headers_with_list_of_tuples_duplicated_names(create_session): ("h1", "v2")]) assert isinstance(headers, CIMultiDict) - assert headers == CIMultiDict([("H2", "header2"), - ("H1", "v1"), - ("H1", "v2")]) + assert list(sorted(headers.items())) == [("h1", "v1"), + ("h1", "v2"), + ("h2", "header2")] def test_http_GET(session, params): diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 4173765870c..3d677bde884 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -66,8 +66,8 @@ def test_parse_headers(parser): assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [('Test', 'line continue'), - ('Test2', 'data')] + assert list(msg.headers.items()) == [('test', 'line continue'), + ('test2', 'data')] assert msg.raw_headers == ((b'test', b'line continue'), (b'test2', b'data')) assert not msg.should_close @@ -134,7 +134,7 @@ def test_headers_multi_feed(parser): assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [('Test', 'line continue')] + assert list(msg.headers.items()) == [('test', 'line continue')] assert msg.raw_headers == ((b'test', b'line continue'),) assert not msg.should_close assert msg.compression is None diff --git a/tests/test_multipart.py b/tests/test_multipart.py index e3dd38ab909..d879adbe139 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -843,7 +843,7 @@ def test_writer_write(buf, stream, writer): b'--:\r\n' b'Content-Type: multipart/mixed; boundary="::"\r\n' - b'X-Custom: test\r\nContent-Length: 93\r\n\r\n' + b'X-CUSTOM: test\r\nContent-Length: 93\r\n\r\n' b'--::\r\n' b'Content-Type: text/plain; charset=utf-8\r\n' b'Content-Length: 14\r\n\r\n' diff --git a/tests/test_web_request.py b/tests/test_web_request.py index 440cfd71e59..36c16823cd0 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -49,7 +49,7 @@ def test_ctor(make_request): assert req.protocol is protocol assert req.transport is protocol.transport assert req.headers == headers - assert req.raw_headers == ((b'Foo', b'bar'),) + assert req.raw_headers == ((b'FOO', b'bar'),) assert req.task is req._task with pytest.warns(DeprecationWarning): @@ -361,7 +361,7 @@ def test_host_by_host_header(make_request): def test_raw_headers(make_request): req = make_request('GET', '/', headers=CIMultiDict({'X-HEADER': 'aaa'})) - assert req.raw_headers == ((b'X-Header', b'aaa'),) + assert req.raw_headers == ((b'X-HEADER', b'aaa'),) def test_rel_url(make_request): From 58a7a58ad2efbde2f48e1b1240da254c64884b95 Mon Sep 17 00:00:00 2001 From: Hu Bo Date: Fri, 23 Jun 2017 01:34:07 +0800 Subject: [PATCH 007/167] Fix #1828: make enable_compression work on HTTP/1.0 (#1910) --- CHANGES.rst | 2 + aiohttp/web_response.py | 67 +++++++++++---- tests/test_client_functional.py | 38 +++++++++ tests/test_web_response.py | 140 +++++++++++++++++++++++++++++++- 4 files changed, 230 insertions(+), 17 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e9ac893d0aa..d90c3042c80 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -141,6 +141,8 @@ Changes - Cancel websocket heartbeat on close #1793 +- Make enable_compression work on HTTP/1.0 #1828 + 2.0.6 (2017-04-04) ------------------ diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index d17e1227efb..1feec8fa089 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -5,6 +5,7 @@ import math import time import warnings +import zlib from email.utils import parsedate from multidict import CIMultiDict, CIMultiDictProxy @@ -299,7 +300,10 @@ def _do_start_compression(self, coding): if coding != ContentCoding.identity: self.headers[hdrs.CONTENT_ENCODING] = coding.value self._payload_writer.enable_compression(coding.value) - self._chunked = True + # Compressed payload may have different content length, + # remove the header + if hdrs.CONTENT_LENGTH in self._headers: + del self._headers[hdrs.CONTENT_LENGTH] def _start_compression(self, request): if self._compression_force: @@ -362,11 +366,14 @@ def _start(self, request, del headers[CONTENT_LENGTH] elif self._length_check: writer.length = self.content_length - if writer.length is None and version >= HttpVersion11: - writer.enable_chunking() - headers[TRANSFER_ENCODING] = 'chunked' - if CONTENT_LENGTH in headers: - del headers[CONTENT_LENGTH] + if writer.length is None: + if version >= HttpVersion11: + writer.enable_chunking() + headers[TRANSFER_ENCODING] = 'chunked' + if CONTENT_LENGTH in headers: + del headers[CONTENT_LENGTH] + else: + keep_alive = False headers.setdefault(CONTENT_TYPE, 'application/octet-stream') headers.setdefault(DATE, request.time_service.strtime()) @@ -489,6 +496,8 @@ def __init__(self, *, body=None, status=200, else: self.body = body + self._compressed_body = None + @property def body(self): return self._body @@ -513,12 +522,10 @@ def body(self, body, headers = self._headers - # enable chunked encoding if needed + # set content-length header if needed if not self._chunked and CONTENT_LENGTH not in headers: size = body.size - if size is None: - self._chunked = True - elif CONTENT_LENGTH not in headers: + if size is not None: headers[CONTENT_LENGTH] = str(size) # set content-type @@ -531,6 +538,8 @@ def body(self, body, if key not in headers: headers[key] = value + self._compressed_body = None + @property def text(self): if self._body is None: @@ -549,6 +558,7 @@ def text(self, text): self._body = text.encode(self.charset) self._body_payload = False + self._compressed_body = None @property def content_length(self): @@ -558,7 +568,13 @@ def content_length(self): if hdrs.CONTENT_LENGTH in self.headers: return super().content_length - if self._body is not None: + if self._compressed_body is not None: + # Return length of the compressed body + return len(self._compressed_body) + elif self._body_payload: + # A payload without content length, or a compressed payload + return None + elif self._body is not None: return len(self._body) else: return 0 @@ -571,7 +587,10 @@ def content_length(self, value): def write_eof(self): if self._eof_sent: return - body = self._body + if self._compressed_body is not None: + body = self._compressed_body + else: + body = self._body if body is not None: if (self._req._method == hdrs.METH_HEAD or self._status in [204, 304]): @@ -586,13 +605,29 @@ def write_eof(self): def _start(self, request): if not self._chunked and hdrs.CONTENT_LENGTH not in self._headers: - if self._body is not None: - self._headers[hdrs.CONTENT_LENGTH] = str(len(self._body)) - else: - self._headers[hdrs.CONTENT_LENGTH] = '0' + if not self._body_payload: + if self._body is not None: + self._headers[hdrs.CONTENT_LENGTH] = str(len(self._body)) + else: + self._headers[hdrs.CONTENT_LENGTH] = '0' return super()._start(request) + def _do_start_compression(self, coding): + if self._body_payload or self._chunked: + return super()._do_start_compression(coding) + if coding != ContentCoding.identity: + # Instead of using _payload_writer.enable_compression, + # compress the whole body + zlib_mode = (16 + zlib.MAX_WBITS + if coding.value == 'gzip' else -zlib.MAX_WBITS) + compressobj = zlib.compressobj(wbits=zlib_mode) + self._compressed_body = compressobj.compress(self._body) +\ + compressobj.flush() + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._headers[hdrs.CONTENT_LENGTH] = \ + str(len(self._compressed_body)) + def json_response(data=sentinel, *, text=None, body=None, status=200, reason=None, headers=None, content_type='application/json', diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 4304c845bb2..d192c339ddb 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -1755,6 +1755,25 @@ def handler(request): resp.close() +@asyncio.coroutine +def test_encoding_deflate_nochunk(loop, test_client): + @asyncio.coroutine + def handler(request): + resp = web.Response(text='text') + resp.enable_compression(web.ContentCoding.deflate) + return resp + + app = web.Application() + app.router.add_get('/', handler) + client = yield from test_client(app) + + resp = yield from client.get('/') + assert 200 == resp.status + txt = yield from resp.text() + assert txt == 'text' + resp.close() + + @asyncio.coroutine def test_encoding_gzip(loop, test_client): @asyncio.coroutine @@ -1775,6 +1794,25 @@ def handler(request): resp.close() +@asyncio.coroutine +def test_encoding_gzip_nochunk(loop, test_client): + @asyncio.coroutine + def handler(request): + resp = web.Response(text='text') + resp.enable_compression(web.ContentCoding.gzip) + return resp + + app = web.Application() + app.router.add_get('/', handler) + client = yield from test_client(app) + + resp = yield from client.get('/') + assert 200 == resp.status + txt = yield from resp.text() + assert txt == 'text' + resp.close() + + @asyncio.coroutine def test_bad_payload_compression(loop, test_client): @asyncio.coroutine diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 8a9b0832c1f..9df6670d555 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -8,6 +8,7 @@ from multidict import CIMultiDict from aiohttp import HttpVersion, HttpVersion10, HttpVersion11, hdrs, signals +from aiohttp.payload import BytesPayload from aiohttp.test_utils import make_mocked_request from aiohttp.web import ContentCoding, Response, StreamResponse, json_response @@ -385,15 +386,152 @@ def test_force_compression_no_accept_gzip(): @asyncio.coroutine -def test_delete_content_length_if_compression_enabled(): +def test_change_content_length_if_compression_enabled(): req = make_request('GET', '/') resp = Response(body=b'answer') resp.enable_compression(ContentCoding.gzip) + yield from resp.prepare(req) + assert resp.content_length is not None and \ + resp.content_length != len(b'answer') + + +@asyncio.coroutine +def test_set_content_length_if_compression_enabled(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH in headers + assert headers[hdrs.CONTENT_LENGTH] == '26' + assert hdrs.TRANSFER_ENCODING not in headers + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', payload_writer=writer) + resp = Response(body=b'answer') + resp.enable_compression(ContentCoding.gzip) + + yield from resp.prepare(req) + assert resp.content_length == 26 + del resp.headers[hdrs.CONTENT_LENGTH] + assert resp.content_length == 26 + + +@asyncio.coroutine +def test_remove_content_length_if_compression_enabled_http11(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH not in headers + assert headers.get(hdrs.TRANSFER_ENCODING, '') == 'chunked' + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', payload_writer=writer) + resp = StreamResponse() + resp.content_length = 123 + resp.enable_compression(ContentCoding.gzip) + yield from resp.prepare(req) + assert resp.content_length is None + + +@asyncio.coroutine +def test_remove_content_length_if_compression_enabled_http10(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH not in headers + assert hdrs.TRANSFER_ENCODING not in headers + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', version=HttpVersion10, + payload_writer=writer) + resp = StreamResponse() + resp.content_length = 123 + resp.enable_compression(ContentCoding.gzip) yield from resp.prepare(req) assert resp.content_length is None +@asyncio.coroutine +def test_force_compression_identity(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH in headers + assert hdrs.TRANSFER_ENCODING not in headers + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', + payload_writer=writer) + resp = StreamResponse() + resp.content_length = 123 + resp.enable_compression(ContentCoding.identity) + yield from resp.prepare(req) + assert resp.content_length == 123 + + +@asyncio.coroutine +def test_force_compression_identity_response(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert headers[hdrs.CONTENT_LENGTH] == "6" + assert hdrs.TRANSFER_ENCODING not in headers + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', + payload_writer=writer) + resp = Response(body=b'answer') + resp.enable_compression(ContentCoding.identity) + yield from resp.prepare(req) + assert resp.content_length == 6 + + +@asyncio.coroutine +def test_remove_content_length_if_compression_enabled_on_payload_http11(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH not in headers + assert headers.get(hdrs.TRANSFER_ENCODING, '') == 'chunked' + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', payload_writer=writer) + payload = BytesPayload(b'answer', headers={"X-Test-Header": "test"}) + resp = Response(body=payload) + assert resp.content_length == 6 + resp.body = payload + resp.enable_compression(ContentCoding.gzip) + yield from resp.prepare(req) + assert resp.content_length is None + + +@asyncio.coroutine +def test_remove_content_length_if_compression_enabled_on_payload_http10(): + writer = mock.Mock() + + def write_headers(status_line, headers): + assert hdrs.CONTENT_LENGTH not in headers + assert hdrs.TRANSFER_ENCODING not in headers + + writer.write_headers.side_effect = write_headers + req = make_request('GET', '/', version=HttpVersion10, + payload_writer=writer) + resp = Response(body=BytesPayload(b'answer')) + resp.enable_compression(ContentCoding.gzip) + yield from resp.prepare(req) + assert resp.content_length is None + + +@asyncio.coroutine +def test_content_length_on_chunked(): + req = make_request('GET', '/') + resp = Response(body=b'answer') + assert resp.content_length == 6 + resp.enable_chunked_encoding() + assert resp.content_length is None + yield from resp.prepare(req) + + @asyncio.coroutine def test_write_non_byteish(): resp = StreamResponse() From 02e3724ec78725248c7582a31b6b60614e422822 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 22 Jun 2017 20:35:01 +0300 Subject: [PATCH 008/167] Fix CHANGES --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d90c3042c80..b863c728e4e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,8 @@ Changes - Improvement for `normalize_path_middleware`. Added possibility to handle URLs with query string. #1995 +- Make enable_compression work on HTTP/1.0 #1828 + - - @@ -141,8 +143,6 @@ Changes - Cancel websocket heartbeat on close #1793 -- Make enable_compression work on HTTP/1.0 #1828 - 2.0.6 (2017-04-04) ------------------ From a05fa9e7f61b6af0170d3a86c9117c5c5baff642 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 22 Jun 2017 21:54:00 +0300 Subject: [PATCH 009/167] Use popall instead of in/del pair (#2002) --- aiohttp/connector.py | 5 ++--- aiohttp/formdata.py | 2 +- aiohttp/web_response.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 2cb2e9d2485..86c44c7358b 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -750,9 +750,8 @@ def _create_proxy_connection(self, req): except OSError as exc: raise ClientProxyConnectionError(*exc.args) from exc - if hdrs.AUTHORIZATION in proxy_req.headers: - auth = proxy_req.headers[hdrs.AUTHORIZATION] - del proxy_req.headers[hdrs.AUTHORIZATION] + auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) + if auth is not None: if not req.ssl: req.headers[hdrs.PROXY_AUTHORIZATION] = auth else: diff --git a/aiohttp/formdata.py b/aiohttp/formdata.py index a5effd7df73..0f2b5ff4a13 100644 --- a/aiohttp/formdata.py +++ b/aiohttp/formdata.py @@ -130,7 +130,7 @@ def _gen_form_data(self): ) # FIXME cgi.FieldStorage doesn't likes body parts with # Content-Length which were sent via chunked transfer encoding - part.headers.pop(hdrs.CONTENT_LENGTH, None) + part.headers.popall(hdrs.CONTENT_LENGTH, None) self._writer.append_payload(part) diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 1feec8fa089..d7d80face49 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -302,8 +302,7 @@ def _do_start_compression(self, coding): self._payload_writer.enable_compression(coding.value) # Compressed payload may have different content length, # remove the header - if hdrs.CONTENT_LENGTH in self._headers: - del self._headers[hdrs.CONTENT_LENGTH] + self._headers.popall(hdrs.CONTENT_LENGTH, None) def _start_compression(self, request): if self._compression_force: From 7849d6dae61d004804a73316596f29dda2a122ba Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 24 Jun 2017 11:53:57 +0300 Subject: [PATCH 010/167] Drop deprecated RequestHandler.finish_connections (#2006) --- CHANGES.rst | 2 +- aiohttp/web_server.py | 2 -- docs/web_reference.rst | 13 ------------- tests/autobahn/server.py | 2 +- tests/test_connector.py | 2 +- tests/test_web_request_handler.py | 8 ++++---- 6 files changed, 7 insertions(+), 22 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b863c728e4e..6356f8c9fe6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,7 +9,7 @@ Changes - Make enable_compression work on HTTP/1.0 #1828 -- +- Drop deprecated `Server.finish_connections` - diff --git a/aiohttp/web_server.py b/aiohttp/web_server.py index f936395df17..b765d27849f 100644 --- a/aiohttp/web_server.py +++ b/aiohttp/web_server.py @@ -45,7 +45,5 @@ def shutdown(self, timeout=None): self._connections.clear() self.time_service.close() - finish_connections = shutdown - def __call__(self): return RequestHandler(self, loop=self._loop, **self._kwargs) diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 48185cf130c..7bac2644b96 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1391,19 +1391,6 @@ A protocol factory compatible with A :ref:`coroutine` that should be called to close all opened connections. - .. coroutinemethod:: Server.finish_connections(timeout) - - .. deprecated:: 1.2 - - A deprecated alias for :meth:`shutdown`. - - .. versionchanged:: 1.2 - - ``Server`` was called ``RequestHandlerFactory`` before ``aiohttp==1.2``. - - The rename has no deprecation period but it's safe: no user - should instantiate the class by hands. - Router ^^^^^^ diff --git a/tests/autobahn/server.py b/tests/autobahn/server.py index 447e152d147..f23b7069306 100644 --- a/tests/autobahn/server.py +++ b/tests/autobahn/server.py @@ -45,7 +45,7 @@ def main(loop): @asyncio.coroutine def finish(app, srv, handler): srv.close() - yield from handler.finish_connections() + yield from handler.shutdown() yield from srv.wait_closed() diff --git a/tests/test_connector.py b/tests/test_connector.py index ccee517743e..6648bed0263 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -1102,7 +1102,7 @@ def setUp(self): def tearDown(self): if self.handler: - self.loop.run_until_complete(self.handler.finish_connections()) + self.loop.run_until_complete(self.handler.shutdown()) self.loop.stop() self.loop.run_forever() self.loop.close() diff --git a/tests/test_web_request_handler.py b/tests/test_web_request_handler.py index b2865b2903d..0285c668045 100644 --- a/tests/test_web_request_handler.py +++ b/tests/test_web_request_handler.py @@ -34,7 +34,7 @@ def test_connections(loop): @asyncio.coroutine -def test_finish_connection_no_timeout(loop): +def test_shutdown_no_timeout(loop): app = web.Application() manager = app.make_handler(loop=loop) @@ -43,7 +43,7 @@ def test_finish_connection_no_timeout(loop): transport = mock.Mock() manager.connection_made(handler, transport) - yield from manager.finish_connections() + yield from manager.shutdown() manager.connection_lost(handler, None) assert manager.connections == [] @@ -51,7 +51,7 @@ def test_finish_connection_no_timeout(loop): @asyncio.coroutine -def test_finish_connection_timeout(loop): +def test_shutdown_timeout(loop): app = web.Application() manager = app.make_handler(loop=loop) @@ -60,7 +60,7 @@ def test_finish_connection_timeout(loop): transport = mock.Mock() manager.connection_made(handler, transport) - yield from manager.finish_connections(timeout=0.1) + yield from manager.shutdown(timeout=0.1) manager.connection_lost(handler, None) assert manager.connections == [] From 671839ecbbe59b20ed70866a369166a2ff513e13 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 24 Jun 2017 14:09:35 +0300 Subject: [PATCH 011/167] Update third party list --- README.rst | 41 +++++++++------- docs/third_party.rst | 109 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 31b4f95c06f..150da560c0c 100644 --- a/README.rst +++ b/README.rst @@ -20,21 +20,28 @@ Async http client/server framework aiohttp 2.0 release! -------------------- -For this release we completely refactored low-level implementation of http handling. -Finally `uvloop` gives performance improvement. Overall performance improvement -should be around 70-90% compared to 1.x version. - -We took opportunity to refactor long standing api design problems across whole package. -Client exceptions handling has been cleaned up and now much more straight forward. Client payload -management simplified and allows to extend with any custom type. Client connection pool -implementation has been redesigned as well, now there is no need for actively releasing response objects, -aiohttp handles connection release automatically. - -Another major change, we moved aiohttp development to public organization https://github.com/aio-libs - -With this amount of api changes we had to make backward incompatible changes. Please check this migration document http://aiohttp.readthedocs.io/en/latest/migration.html - -Please report problems or annoyance with with api to https://github.com/aio-libs/aiohttp +For this release we completely refactored low-level implementation of +http handling. Finally `uvloop` gives performance +improvement. Overall performance improvement should be around 70-90% +compared to 1.x version. + +We took opportunity to refactor long standing api design problems +across whole package. Client exceptions handling has been cleaned up +and now much more straight forward. Client payload management +simplified and allows to extend with any custom type. Client +connection pool implementation has been redesigned as well, now there +is no need for actively releasing response objects, aiohttp handles +connection release automatically. + +Another major change, we moved aiohttp development to public +organization https://github.com/aio-libs + +With this amount of api changes we had to make backward incompatible +changes. Please check this migration document +http://aiohttp.readthedocs.io/en/latest/migration.html + +Please report problems or annoyance with with api to +https://github.com/aio-libs/aiohttp Features @@ -161,7 +168,9 @@ License Keepsafe -------- -The aiohttp community would like to thank Keepsafe (https://www.getkeepsafe.com) for it's support in the early days of the project. +The aiohttp community would like to thank Keepsafe +(https://www.getkeepsafe.com) for it's support in the early days of +the project. Source code diff --git a/docs/third_party.rst b/docs/third_party.rst index fb158e9520d..39dbe5110ca 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -32,11 +32,28 @@ aiohttp extensions provides sessions for :mod:`aiohttp.web`. - `aiohttp-debugtoolbar `_ - is a library for *debug toolbar* support for :mod:`aiohttp.web`. + is a library for *debug toolbar* support for :mod:`aiohttp.web`. - `aiohttp-security `_ - auth and permissions for :mod:`aiohttp.web`. + auth and permissions for :mod:`aiohttp.web`. +- `aiohttp-devtools `_ + provides development tools for :mod:`aiohttp.web` applications. + +- `aiohttp-cors `_ CORS + support for aiohttp. + +- `aiohttp-sse `_ Server-sent + events support for aiohttp. + +- `pytest-aiohttp `_ + pytest plugin for aiohttp support. + +- `aiohttp-mako `_ mako + template renderer for aiohttp.web. + +- `aiohttp-jinja2 `_ Jinja2 + template renderer for aiohttp.web. Database drivers ^^^^^^^^^^^^^^^^ @@ -47,6 +64,15 @@ Database drivers - `aioredis `_ Redis async driver. +Other tools +^^^^^^^^^^^ + +- `aiodocker `_ Python Docker + API client based on asyncio and aiohttp. + +- `aiobotocore `_ asyncio + support for botocore library using aiohttp. + Approved third-party libraries ------------------------------ @@ -80,8 +106,83 @@ period ask to raise he status. - `aiohttp-cache `_ A cache system for aiohttp server. + - `aiocache `_ Caching for asyncio with multiple backends (framework agnostic) -- `aiohttp-devtools `_ - provides development tools for :mod:`aiohttp.web` applications +- `gain `_ Web crawling framework + based on asyncio for everyone. + +- `aiohttp-swagger `_ + Swagger API Documentation builder for aiohttp server. + +- `raven-aiohttp `_ An + aiohttp transport for raven-python (Sentry client). + +- `webargs `_ A friendly library + for parsing HTTP request arguments, with built-in support for + popular web frameworks, including Flask, Django, Bottle, Tornado, + Pyramid, webapp2, Falcon, and aiohttp. + +- `aioauth-client `_ OAuth + client for aiohttp. + +- `aiohttpretty + `_ A simple + asyncio compatible httpretty mock using aiohttp. + +- `aioresponses `_ + Aioresponses is a helper for mock/fake web requests in python + aiohttp package. + +- `aiohttp-transmute + `_ A transmute + implementation for aiohttp. + +- `aiohttp_apiset `_ + Package to build routes using swagger specification. + +- `aiohttp-login `_ + Registration and authorization (including social) for aiohttp apps. + +- `aiohttp_utils `_ Handy + utilities for building aiohttp.web applications. + +- `aiohttpproxy `_ Simple + aiohttp HTTP proxy. + +- `aiohttp_traversal `_ + Traversal based router for aiohttp.web. + +- `aiohttp_autoreload + `_ Makes aiohttp + server autoreload on source code change. + +- `gidgethub `_ An async + GitHub API library for Python. + +- `aiohttp_jrpc `_ aiohttp + jsonrpc service. + +- `fbemissary `_ A bot + framework for the Facebook Messenger platform, built on asyncio and + aiohttp. + +- `aioslacker `_ slacker + wrapper for asyncio. + +- `aioreloader `_ Port of + tornado reloader to asyncio. + +- `aiohttp_babel `_ Babel + localisation support for aiohttp. + +- `python-mocket `_ a + socket mock framework - for all kinds of socket animals, web-clients + included. + +- `aioraft `_ asyncio RAFT + algorithm bassed on aiohttp. + +- `home-assistant `_ + Open-source home automation platform running on Python 3. From 38fefde999142a2ef38cfa2974a29dd1328499c8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 24 Jun 2017 14:53:34 +0300 Subject: [PATCH 012/167] Fix spelling --- docs/spelling_wordlist.txt | 6 ++++++ docs/third_party.rst | 18 +++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 965027ef58b..f8c35bcff5d 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -24,6 +24,7 @@ BasicAuth BodyPartReader builtin BytesIO +botocore cchardet cChardet charset @@ -62,6 +63,7 @@ env environ eof epoll +Facebook fallback filename finalizers @@ -78,6 +80,7 @@ highlevel hostnames HTTPException HttpProcessingError +httpretty https incapsulates impl @@ -106,6 +109,7 @@ localhost login lookup lookups +Mako manylinux metadata middleware @@ -128,6 +132,7 @@ Nginx Nikolay noop nowait +OAuth optimizations os outcoming @@ -221,6 +226,7 @@ utils uvloop vcvarsall waituntil +webapp websocket websockets Websockets diff --git a/docs/third_party.rst b/docs/third_party.rst index 39dbe5110ca..b27d7e0ee60 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -49,7 +49,7 @@ aiohttp extensions - `pytest-aiohttp `_ pytest plugin for aiohttp support. -- `aiohttp-mako `_ mako +- `aiohttp-mako `_ Mako template renderer for aiohttp.web. - `aiohttp-jinja2 `_ Jinja2 @@ -131,9 +131,8 @@ period ask to raise he status. `_ A simple asyncio compatible httpretty mock using aiohttp. -- `aioresponses `_ - Aioresponses is a helper for mock/fake web requests in python - aiohttp package. +- `aioresponses `_ a + helper for mock/fake web requests in python aiohttp package. - `aiohttp-transmute `_ A transmute @@ -143,7 +142,8 @@ period ask to raise he status. Package to build routes using swagger specification. - `aiohttp-login `_ - Registration and authorization (including social) for aiohttp apps. + Registration and authorization (including social) for aiohttp + applications. - `aiohttp_utils `_ Handy utilities for building aiohttp.web applications. @@ -156,13 +156,13 @@ period ask to raise he status. - `aiohttp_autoreload `_ Makes aiohttp - server autoreload on source code change. + server auto-reload on source code change. - `gidgethub `_ An async GitHub API library for Python. - `aiohttp_jrpc `_ aiohttp - jsonrpc service. + JSON-RPC service. - `fbemissary `_ A bot framework for the Facebook Messenger platform, built on asyncio and @@ -175,14 +175,14 @@ period ask to raise he status. tornado reloader to asyncio. - `aiohttp_babel `_ Babel - localisation support for aiohttp. + localization support for aiohttp. - `python-mocket `_ a socket mock framework - for all kinds of socket animals, web-clients included. - `aioraft `_ asyncio RAFT - algorithm bassed on aiohttp. + algorithm based on aiohttp. - `home-assistant `_ Open-source home automation platform running on Python 3. From 8cc4cd5e385374e6ab1c85346f0169d023828baa Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Jun 2017 14:35:26 +0300 Subject: [PATCH 013/167] Fix #1997: Use towncrier for changelog generation (#2010) * Adopt towncrier * Add missing files * Mention itself in changes * Fix settings * Adopt CHANGES formatting to towncrier generator * Change title format * Fix title level consistency --- CHANGES.rst | 58 ++++------- HISTORY.rst | 202 ++++++++++++++++++++------------------- PULL_REQUEST_TEMPLATE.md | 13 ++- README.rst | 25 ++--- changes/.gitignore | 1 + changes/1828.feature | 1 + changes/1995.feature | 2 + changes/1997.feature | 1 + changes/2006.removal | 1 + pyproject.toml | 5 + requirements-dev.txt | 1 + 11 files changed, 159 insertions(+), 151 deletions(-) create mode 100644 changes/.gitignore create mode 100644 changes/1828.feature create mode 100644 changes/1995.feature create mode 100644 changes/1997.feature create mode 100644 changes/2006.removal create mode 100644 pyproject.toml diff --git a/CHANGES.rst b/CHANGES.rst index 6356f8c9fe6..9b52edd5e92 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,38 +1,21 @@ +======= Changes ======= -2.3.0 (2017-xx-xx) ------------------- - -- Improvement for `normalize_path_middleware`. Added possibility to handle - URLs with query string. #1995 - -- Make enable_compression work on HTTP/1.0 #1828 - -- Drop deprecated `Server.finish_connections` - -- - -- +.. + You should *NOT* be adding new change log entries to this file, this + file is managed by towncrier. You *may* edit previous change logs to + fix problems like typo corrections or such. + To add a new change log entry, please see + https://pip.pypa.io/en/latest/development/#adding-a-news-entry + we named the news folder "changes". -- +.. towncrier release notes start -- -- - -- - -- - -- - -- - -- 2.2.0 (2017-06-20) ------------------- +================== - Add doc for add_head, update doc for add_get. #1944 @@ -77,7 +60,7 @@ Changes 2.1.0 (2017-05-26) ------------------- +================== - Added support for experimental `async-tokio` event loop written in Rust https://github.com/PyO3/tokio @@ -134,7 +117,8 @@ Changes 2.0.7 (2017-04-12) ------------------- +================== + - Fix *pypi* distribution - Fix exception description #1807 @@ -145,7 +129,7 @@ Changes 2.0.6 (2017-04-04) ------------------- +================== - Keeping blank values for `request.post()` and `multipart.form()` #1765 @@ -156,7 +140,7 @@ Changes 2.0.5 (2017-03-29) ------------------- +================== - Memory leak with aiohttp.request #1756 @@ -167,7 +151,7 @@ Changes 2.0.4 (2017-03-27) ------------------- +================== - Memory leak with aiohttp.request #1756 @@ -177,7 +161,7 @@ Changes 2.0.3 (2017-03-24) ------------------- +================== - Call https website through proxy will cause error #1745 @@ -185,7 +169,7 @@ Changes 2.0.2 (2017-03-21) ------------------- +================== - Fixed Application.on_loop_available signal #1739 @@ -193,7 +177,7 @@ Changes 2.0.1 (2017-03-21) ------------------- +================== - Fix allow-head to include name on route #1737 @@ -201,7 +185,7 @@ Changes 2.0.0 (2017-03-20) ------------------- +================== - Added `json` to `ClientSession.request()` method #1726 @@ -217,7 +201,7 @@ Changes `2.0.0rc1` (2017-03-15) ------------------------ +======================= - Properly handle payload errors #1710 diff --git a/HISTORY.rst b/HISTORY.rst index 7f71ffbb0ea..c2f5b6fcf47 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,11 +1,11 @@ 1.3.5 (2017-03-16) ------------------- +================== - Fixed None timeout support #1720 1.3.4 (2017-03-14) ------------------- +================== - Revert timeout handling in client request @@ -25,13 +25,13 @@ 1.3.3 (2017-02-19) ------------------- +================== - Fixed memory leak in time service #1656 1.3.2 (2017-02-16) ------------------- +================== - Awaiting on WebSocketResponse.send_* does not work #1645 @@ -41,7 +41,7 @@ 1.3.1 (2017-02-09) ------------------- +================== - Handle CLOSING in WebSocketResponse.__anext__ @@ -49,9 +49,10 @@ 1.3.0 (2017-02-08) ------------------- +================== -- Multipart writer validates the data on append instead of on a request send #920 +- Multipart writer validates the data on append instead of on a + request send #920 - Multipart reader accepts multipart messages with or without their epilogue to consistently handle valid and legacy behaviors #1526 #1581 @@ -75,9 +76,11 @@ - Having a `:` or `@` in a route does not work #1552 -- Added `receive_timeout` timeout for websocket to receive complete message. #1325 +- Added `receive_timeout` timeout for websocket to receive complete + message. #1325 -- Added `heartbeat` parameter for websocket to automatically send `ping` message. #1024 #777 +- Added `heartbeat` parameter for websocket to automatically send + `ping` message. #1024 #777 - Remove `web.Application` dependency from `web.UrlDispatcher` #1510 @@ -109,15 +112,16 @@ - Close response connection if we can not consume whole http message during client response release -- Abort closed ssl client transports, broken servers can keep socket open un-limit time #1568 +- Abort closed ssl client transports, broken servers can keep socket + open un-limit time #1568 - Log warning instead of `RuntimeError` is websocket connection is closed. - Deprecated: `aiohttp.protocol.HttpPrefixParser` will be removed in 1.4 #1590 -- Deprecated: Servers response's `.started`, `.start()` and `.can_start()` method - will be removed in 1.4 #1591 +- Deprecated: Servers response's `.started`, `.start()` and + `.can_start()` method will be removed in 1.4 #1591 - Deprecated: Adding `sub app` via `app.router.add_subapp()` is deprecated use `app.add_subapp()` instead, will be removed in 1.4 #1592 @@ -131,7 +135,7 @@ 1.2.0 (2016-12-17) ------------------- +================== - Extract `BaseRequest` from `web.Request`, introduce `web.Server` (former `RequestHandlerFactory`), introduce new low-level web server @@ -190,18 +194,18 @@ 1.1.6 (2016-11-28) ------------------- +================== - Fix `BodyPartReader.read_chunk` bug about returns zero bytes before `EOF` #1428 1.1.5 (2016-11-16) ------------------- +================== - Fix static file serving in fallback mode #1401 1.1.4 (2016-11-14) ------------------- +================== - Make `TestServer.make_url` compatible with `yarl.URL` #1389 @@ -210,13 +214,13 @@ 1.1.3 (2016-11-10) ------------------- +================== - Support *root* resources for sub-applications #1379 1.1.2 (2016-11-08) ------------------- +================== - Allow starting variables with an underscore #1379 @@ -227,12 +231,12 @@ - Don't propagate pre and post signals to sub-application #1377 1.1.1 (2016-11-04) ------------------- +================== - Fix documentation generation #1120 1.1.0 (2016-11-03) ------------------- +================== - Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) @@ -332,21 +336,21 @@ - Implement `web.Request.clone()` #1361 1.0.5 (2016-10-11) ------------------- +================== - Fix StreamReader._read_nowait to return all available data up to the requested amount #1297 1.0.4 (2016-09-22) ------------------- +================== - Fix FlowControlStreamReader.read_nowait so that it checks whether the transport is paused #1206 1.0.2 (2016-09-22) ------------------- +================== - Make CookieJar compatible with 32-bit systems #1188 @@ -358,7 +362,7 @@ 1.0.1 (2016-09-16) ------------------- +================== - Restore `aiohttp.web.MsgType` alias for `aiohttp.WSMsgType` for sake of backward compatibility #1178 @@ -377,7 +381,7 @@ 1.0.0 (2016-09-16) -------------------- +================== - Change default size for client session's connection pool from unlimited to 20 #977 @@ -519,18 +523,18 @@ 0.22.5 (08-02-2016) -------------------- +=================== - Pin miltidict version to >=1.2.2 0.22.3 (07-26-2016) -------------------- +=================== - Do not filter cookies if unsafe flag provided #1005 0.22.2 (07-23-2016) -------------------- +=================== - Suppress CancelledError when Timeout raises TimeoutError #970 @@ -544,14 +548,14 @@ 0.22.1 (07-16-2016) -------------------- +=================== - Large cookie expiration/max-age does not break an event loop from now (fixes #967) 0.22.0 (07-15-2016) -------------------- +=================== - Fix bug in serving static directory #803 @@ -648,18 +652,18 @@ - Dup a socket for sendfile usage #964 0.21.6 (05-05-2016) -------------------- +=================== - Drop initial query parameters on redirects #853 0.21.5 (03-22-2016) -------------------- +=================== - Fix command line arg parsing #797 0.21.4 (03-12-2016) -------------------- +=================== - Fix ResourceAdapter: don't add method to allowed if resource is not match #826 @@ -667,13 +671,13 @@ - Fix Resource: append found method to returned allowed methods 0.21.2 (02-16-2016) -------------------- +=================== - Fix a regression: support for handling ~/path in static file routes was broken #782 0.21.1 (02-10-2016) -------------------- +=================== - Make new resources classes public #767 @@ -682,7 +686,7 @@ - Fix cmd-line parameter names in doc 0.21.0 (02-04-2016) --------------------- +=================== - Introduce on_shutdown signal #722 @@ -756,7 +760,7 @@ - Add local socket binding for TCPConnector #678 0.20.2 (01-07-2016) --------------------- +=================== - Enable use of `await` for a class based view #717 @@ -765,7 +769,7 @@ - Fix memory leak in headers processing (thanks to Marco Paolini) #723 0.20.1 (12-30-2015) -------------------- +=================== - Raise RuntimeError is Timeout context manager was used outside of task context. @@ -776,7 +780,7 @@ 0.20.0 (12-28-2015) -------------------- +=================== - Extend list of web exceptions, add HTTPMisdirectedRequest, HTTPUpgradeRequired, HTTPPreconditionRequired, HTTPTooManyRequests, @@ -827,7 +831,7 @@ 0.19.0 (11-25-2015) -------------------- +=================== - Memory leak in ParserBuffer #579 @@ -875,29 +879,29 @@ - Add Timeout context manager #611 0.18.4 (13-11-2015) -------------------- +=================== - Relax rule for router names again by adding dash to allowed characters: they may contain identifiers, dashes, dots and columns 0.18.3 (25-10-2015) -------------------- +=================== - Fix formatting for _RequestContextManager helper #590 0.18.2 (22-10-2015) -------------------- +=================== - Fix regression for OpenSSL < 1.0.0 #583 0.18.1 (20-10-2015) -------------------- +=================== - Relax rule for router names: they may contain dots and columns starting from now 0.18.0 (19-10-2015) -------------------- +=================== - Use errors.HttpProcessingError.message as HTTP error reason and message #459 @@ -974,14 +978,14 @@ - Added `async for` support for aiohttp stream #542 0.17.4 (09-29-2015) -------------------- +=================== - Properly parse URL path in aiohttp.web.Request #489 - Add missing coroutine decorator, the client api is await-compatible now 0.17.3 (08-28-2015) ---------------------- +=================== - Remove Content-Length header on compressed responses #450 @@ -992,19 +996,19 @@ - Fix connection pooling #473 0.17.2 (08-11-2015) ---------------------- +=================== - Don't forget to pass `data` argument forward #462 - Fix multipart read bytes count #463 0.17.1 (08-10-2015) ---------------------- +=================== - Fix multidict comparison to arbitrary abc.Mapping 0.17.0 (08-04-2015) ---------------------- +=================== - Make StaticRoute support Last-Modified and If-Modified-Since headers #386 @@ -1059,31 +1063,31 @@ 0.16.6 (07-15-2015) -------------------- +=================== - Skip compilation on Windows if vcvarsall.bat cannot be found #438 0.16.5 (06-13-2015) -------------------- +=================== - Get rid of all comprehensions and yielding in _multidict #410 0.16.4 (06-13-2015) -------------------- +=================== - Don't clear current exception in multidict's `__repr__` (cythonized versions) #410 0.16.3 (05-30-2015) -------------------- +=================== - Fix StaticRoute vulnerability to directory traversal attacks #380 0.16.2 (05-27-2015) -------------------- +=================== - Update python version required for `__del__` usage: it's actually 3.4.1 instead of 3.4.0 @@ -1093,12 +1097,12 @@ 0.16.1 (05-27-2015) -------------------- +=================== - Fix regression in static file handling #377 0.16.0 (05-26-2015) -------------------- +=================== - Unset waiter future after cancellation #363 @@ -1142,7 +1146,7 @@ 0.15.3 (04-22-2015) -------------------- +=================== - Fix graceful shutdown handling @@ -1150,7 +1154,7 @@ 0.15.2 (04-19-2015) -------------------- +=================== - Flow control subsystem refactoring @@ -1172,7 +1176,7 @@ 0.15.1 (03-31-2015) -------------------- +=================== - Pass Autobahn Testsuite tests @@ -1188,7 +1192,7 @@ 0.15.0 (03-27-2015) -------------------- +=================== - Client WebSockets support @@ -1214,19 +1218,19 @@ 0.14.4 (01-29-2015) -------------------- +=================== - Fix issue with error during constructing of url with regex parts #264 0.14.3 (01-28-2015) -------------------- +=================== - Use path='/' by default for cookies #261 0.14.2 (01-23-2015) -------------------- +=================== - Connections leak in BaseConnector #253 @@ -1236,7 +1240,7 @@ 0.14.1 (01-15-2015) -------------------- +=================== - HttpMessage._add_default_headers does not overwrite existing headers #216 @@ -1284,7 +1288,7 @@ 0.13.1 (12-31-2014) --------------------- +=================== - Add `aiohttp.web.StreamResponse.started` property #213 @@ -1295,7 +1299,7 @@ 0.13.0 (12-29-2014) -------------------- +=================== - `StreamResponse.charset` converts value to lower-case on assigning. @@ -1314,7 +1318,7 @@ 0.12.0 (12-12-2014) -------------------- +=================== - Deep refactoring of `aiohttp.web` in backward-incompatible manner. Sorry, we have to do this. @@ -1347,7 +1351,7 @@ 0.11.0 (11-29-2014) -------------------- +=================== - Support named routes in `aiohttp.web.UrlDispatcher` #179 @@ -1355,13 +1359,13 @@ 0.10.2 (11-19-2014) -------------------- +=================== - Don't unquote `environ['PATH_INFO']` in wsgi.py #177 0.10.1 (11-17-2014) -------------------- +=================== - aiohttp.web.HTTPException and descendants now files response body with string like `404: NotFound` @@ -1371,7 +1375,7 @@ 0.10.0 (11-13-2014) -------------------- +=================== - Add aiohttp.web subpackage for highlevel HTTP server support. @@ -1389,13 +1393,13 @@ 0.9.3 (10-30-2014) ------------------- +================== - Fix compatibility with asyncio 3.4.1+ #170 0.9.2 (10-16-2014) ------------------- +================== - Improve redirect handling #157 @@ -1405,7 +1409,7 @@ 0.9.1 (08-30-2014) ------------------- +================== - Added MultiDict support for client request params and data #114. @@ -1419,7 +1423,7 @@ 0.9.0 (07-08-2014) ------------------- +================== - Better client basic authentication support #112. @@ -1434,13 +1438,13 @@ 0.8.4 (07-04-2014) ------------------- +================== - Change ProxyConnector authorization parameters. 0.8.3 (07-03-2014) ------------------- +================== - Publish TCPConnector properties: verify_ssl, family, resolve, resolved_hosts. @@ -1450,7 +1454,7 @@ 0.8.2 (06-22-2014) ------------------- +================== - Make ProxyConnector.proxy immutable property. @@ -1462,7 +1466,7 @@ 0.8.1 (06-18-2014) ------------------- +================== - Use case insensitive multidict for server request/response headers. @@ -1476,7 +1480,7 @@ 0.8.0 (06-06-2014) ------------------- +================== - Add support for utf-8 values in HTTP headers @@ -1494,13 +1498,13 @@ 0.7.3 (05-20-2014) ------------------- +================== - Simple HTTP proxy support. 0.7.2 (05-14-2014) ------------------- +================== - Get rid of `__del__` methods @@ -1508,7 +1512,7 @@ 0.7.1 (04-28-2014) ------------------- +================== - Do not unquote client request urls. @@ -1521,7 +1525,7 @@ 0.7.0 (04-16-2014) ------------------- +================== - Connection flow control. @@ -1531,7 +1535,7 @@ 0.6.5 (03-29-2014) ------------------- +================== - Added client session reuse timeout. @@ -1543,13 +1547,13 @@ 0.6.4 (02-27-2014) ------------------- +================== - Log content-length missing warning only for put and post requests. 0.6.3 (02-27-2014) ------------------- +================== - Better support for server exit. @@ -1557,7 +1561,7 @@ 0.6.2 (02-18-2014) ------------------- +================== - Fix trailing char in allowed_methods. @@ -1565,7 +1569,7 @@ 0.6.1 (02-17-2014) ------------------- +================== - Added utility method HttpResponse.read_and_close() @@ -1575,13 +1579,13 @@ 0.6.0 (02-12-2014) ------------------- +================== - Better handling for process exit. 0.5.0 (01-29-2014) ------------------- +================== - Allow to use custom HttpRequest client class. @@ -1593,20 +1597,20 @@ 0.4.4 (11-15-2013) ------------------- +================== - Resolve only AF_INET family, because it is not clear how to pass extra info to asyncio. 0.4.3 (11-15-2013) ------------------- +================== - Allow to wait completion of request with `HttpResponse.wait_for_close()` 0.4.2 (11-14-2013) ------------------- +================== - Handle exception in client request stream. @@ -1614,13 +1618,13 @@ 0.4.1 (11-12-2013) ------------------- +================== - Added client support for `expect: 100-continue` header. 0.4 (11-06-2013) ----------------- +================ - Added custom wsgi application close procedure @@ -1628,7 +1632,7 @@ 0.3 (11-04-2013) ----------------- +================ - Added PortMapperWorker @@ -1642,6 +1646,6 @@ 0.2 ---- +=== - Fix packaging diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 084e7ad2a4e..548a794a308 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -28,6 +28,13 @@ - [ ] If you provide code modification, please add yourself to `CONTRIBUTORS.txt` * The format is <Name> <Surname>. * Please keep alphabetical order, the file is sorted by names. -- [ ] Add a new entry to `CHANGES.rst` - * Choose any open position to avoid merge conflicts with other PRs. - * Add a link to the issue you are fixing (if any) using `#issue_number` format at the end of changelog message. Use Pull Request number if there are no issues for PR or PR covers the issue only partially. +- [ ] Add a new news fragment into the `changes` folder + * name it `.` for example (588.bug) + * if you don't have an `issue_id` change it to the pr id after creating the pr + * ensure type is one of the following: + * `.feature`: Signifying a new feature. + * `.bugfix`: Signifying a bug fix. + * `.doc`: Signifying a documentation improvement. + * `.removal`: Signifying a deprecation or removal of public API. + * `.misc`: A ticket has been closed, but it is not of interest to users. + * Make sure to use full sentences with correct case and punctuation, for example: "Fix issue with non-ascii contents in doctest text files." diff --git a/README.rst b/README.rst index 150da560c0c..f9e3a507653 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,4 @@ +================================== Async http client/server framework ================================== @@ -18,7 +19,7 @@ Async http client/server framework aiohttp 2.0 release! --------------------- +==================== For this release we completely refactored low-level implementation of http handling. Finally `uvloop` gives performance @@ -45,7 +46,7 @@ https://github.com/aio-libs/aiohttp Features --------- +======== - Supports both client and server side of HTTP protocol. - Supports both client and server Web-Sockets out-of-the-box. @@ -53,10 +54,10 @@ Features Getting started ---------------- +=============== Client -^^^^^^ +------ To retrieve something from the web: @@ -81,7 +82,7 @@ To retrieve something from the web: Server -^^^^^^ +------ This is simple usage example: @@ -131,17 +132,17 @@ should be replaced by:: ret = yield from f() Documentation -------------- +============= https://aiohttp.readthedocs.io/ Discussion list ---------------- +=============== *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs Requirements ------------- +============ - Python >= 3.4.2 - async-timeout_ @@ -160,13 +161,13 @@ recommended for sake of speed). .. _cChardet: https://pypi.python.org/pypi/cchardet License -------- +======= ``aiohttp`` is offered under the Apache 2 license. Keepsafe --------- +======== The aiohttp community would like to thank Keepsafe (https://www.getkeepsafe.com) for it's support in the early days of @@ -174,13 +175,13 @@ the project. Source code ------------- +=========== The latest developer version is available in a github repository: https://github.com/aio-libs/aiohttp Benchmarks ----------- +========== If you are interested in by efficiency, AsyncIO community maintains a list of benchmarks on the official wiki: diff --git a/changes/.gitignore b/changes/.gitignore new file mode 100644 index 00000000000..f935021a8f8 --- /dev/null +++ b/changes/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/changes/1828.feature b/changes/1828.feature new file mode 100644 index 00000000000..d3e8fcfde21 --- /dev/null +++ b/changes/1828.feature @@ -0,0 +1 @@ +Make enable_compression work on HTTP/1.0 diff --git a/changes/1995.feature b/changes/1995.feature new file mode 100644 index 00000000000..c8dfbf0750d --- /dev/null +++ b/changes/1995.feature @@ -0,0 +1,2 @@ +Improvement for `normalize_path_middleware`. +Added possibility to handle URLs with query string. diff --git a/changes/1997.feature b/changes/1997.feature new file mode 100644 index 00000000000..eb972cf98f1 --- /dev/null +++ b/changes/1997.feature @@ -0,0 +1 @@ +Use towncrier for CHANGES.txt build diff --git a/changes/2006.removal b/changes/2006.removal new file mode 100644 index 00000000000..9cc10f803dd --- /dev/null +++ b/changes/2006.removal @@ -0,0 +1 @@ +Drop deprecated `Server.finish_connections` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..5cb61bb5272 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.towncrier] +package = "aiohttp" +filename = "CHANGES.rst" +directory = "changes/" +title_format = "{version} ({project_date})" diff --git a/requirements-dev.txt b/requirements-dev.txt index 491b0e68c40..6b7ec387ccd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 aiodns==1.1.1 +towncrier==17.4.0 From afaa29f637939401c23c28e6e687619c9cdf71c1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Jun 2017 14:39:11 +0300 Subject: [PATCH 014/167] Mention update to multidict 3.0 in changelog --- changes/1994.feature | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/1994.feature diff --git a/changes/1994.feature b/changes/1994.feature new file mode 100644 index 00000000000..de6b89fa6fb --- /dev/null +++ b/changes/1994.feature @@ -0,0 +1,4 @@ +Switch to `multidict 3.0`. + +All HTTP headers preserve casing now but compared in case-insensitive +way. From 81adb9402162080f2719839ea5cb11c6dafa24e2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Jun 2017 19:28:06 +0300 Subject: [PATCH 015/167] Check ./changes in tests (#2012) --- Makefile | 4 +++- tools/check_changes.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100755 tools/check_changes.py diff --git a/Makefile b/Makefile index 1c0f61933c0..6da664c41ec 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,10 @@ flake: .flake fi @touch .flake +check_changes: + @./tools/check_changes.py -.develop: .install-deps $(shell find aiohttp -type f) .flake +.develop: .install-deps $(shell find aiohttp -type f) .flake check_changes @pip install -e . @touch .develop diff --git a/tools/check_changes.py b/tools/check_changes.py new file mode 100755 index 00000000000..b395a9b0390 --- /dev/null +++ b/tools/check_changes.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import sys +from pathlib import Path + +ALLOWED_SUFFIXES = ['.feature', + '.bugfix', + '.doc', + '.removal', + '.misc'] + + +def get_root(script_path): + folder = script_path.absolute().parent + while not (folder / '.git').exists(): + folder = folder.parent + if folder == folder.anchor: + raise RuntimeError("git repo not found") + return folder + + +def main(argv): + print('Check "changes" folder... ', end='', flush=True) + here = Path(argv[0]) + root = get_root(here) + changes = root / 'changes' + failed = False + for fname in changes.iterdir(): + if fname.name == '.gitignore': + continue + if fname.suffix not in ALLOWED_SUFFIXES: + if not failed: + print('') + print(fname, 'has illegal suffix', file=sys.stderr) + failed = True + + if failed: + print('', file=sys.stderr) + print('Allowed suffixes are:', ALLOWED_SUFFIXES, file=sys.stderr) + print('', file=sys.stderr) + else: + print('OK') + + return int(failed) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From 81e0592139509147863488b021f5381ef3a96fb7 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 25 Jun 2017 19:38:57 +0300 Subject: [PATCH 016/167] Add a script for dropping merged branches --- tools/drop_merged_branches.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 tools/drop_merged_branches.sh diff --git a/tools/drop_merged_branches.sh b/tools/drop_merged_branches.sh new file mode 100755 index 00000000000..d4f315a8987 --- /dev/null +++ b/tools/drop_merged_branches.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +git remote prune origin From 2bd8c55bd50a874d01b97a0c916d3ae499a5d5a0 Mon Sep 17 00:00:00 2001 From: Claudiu Popa Date: Sun, 25 Jun 2017 20:34:09 +0300 Subject: [PATCH 017/167] Fix #2009: `RedirectURLError` is raised instead of RuntimeError (#2011) Raise `RedirectURLError` if a redirect response has no Location or URI HTTP header. Close #2009 --- CONTRIBUTORS.txt | 1 + aiohttp/client.py | 3 ++- aiohttp/http_exceptions.py | 4 ++++ changes/2009.feature | 3 +++ tests/test_client_functional.py | 12 +++++++----- 5 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 changes/2009.feature diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 3d81394e374..40e9f3ca9a6 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -44,6 +44,7 @@ Chris AtLee Chris Laws Chris Moore Christopher Schmitt +Claudiu Popa Damien Nadé Daniel García Daniel Nelson diff --git a/aiohttp/client.py b/aiohttp/client.py index 031602acfa8..0afafddcac6 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -26,6 +26,7 @@ from .helpers import (PY_35, CeilTimeout, TimeoutHandle, deprecated_noop, sentinel) from .http import WS_KEY, WebSocketReader, WebSocketWriter +from .http_exceptions import RedirectURLError from .streams import FlowControlDataQueue @@ -274,7 +275,7 @@ def _request(self, method, url, *, r_url = (resp.headers.get(hdrs.LOCATION) or resp.headers.get(hdrs.URI)) if r_url is None: - raise RuntimeError( + raise RedirectURLError( "{0.method} {0.url} returns " "a redirect [{0.status}] status " "but response lacks a Location " diff --git a/aiohttp/http_exceptions.py b/aiohttp/http_exceptions.py index dc2d1095f93..de42e2d19c3 100644 --- a/aiohttp/http_exceptions.py +++ b/aiohttp/http_exceptions.py @@ -57,6 +57,10 @@ class ContentLengthError(PayloadEncodingError): """Not enough data for satisfy content length header.""" +class RedirectURLError(BadHttpMessage): + """A redirect response lacks a Location or URI HTTP header""" + + class LineTooLong(BadHttpMessage): def __init__(self, line, limit='Unknown'): diff --git a/changes/2009.feature b/changes/2009.feature new file mode 100644 index 00000000000..eb4dd60077a --- /dev/null +++ b/changes/2009.feature @@ -0,0 +1,3 @@ +`RedirectURLError` is raised instead of RuntimeError + +Raise `RedirectURLError` if a redirect response has no Location or URI HTTP header. \ No newline at end of file diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index d192c339ddb..c1a550dd699 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -15,6 +15,7 @@ from aiohttp import hdrs, web from aiohttp.client import ServerFingerprintMismatch from aiohttp.helpers import create_future +from aiohttp.http_exceptions import RedirectURLError from aiohttp.multipart import MultipartWriter @@ -2106,12 +2107,13 @@ def handler_redirect(request): app.router.add_route('GET', '/redirect', handler_redirect) client = yield from test_client(app) - with pytest.raises(RuntimeError) as ctx: + with pytest.raises(RedirectURLError) as ctx: yield from client.get('/redirect') - assert str(ctx.value) == ('GET http://127.0.0.1:{}/redirect returns ' - 'a redirect [301] status but response lacks ' - 'a Location or URI HTTP header' - .format(client.port)) + expected_msg = ('GET http://127.0.0.1:{}/redirect returns ' + 'a redirect [301] status but response lacks ' + 'a Location or URI HTTP header' + .format(client.port)) + assert str(ctx.value.message) == expected_msg @asyncio.coroutine From 00f5c40533b3095c31c6502bdea0e0d46a08fec7 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 10:51:33 +0300 Subject: [PATCH 018/167] Fix #2014: Drop pytest-timeout plugin --- CONTRIBUTING.rst | 17 +++++------------ changes/2014.misc | 1 + requirements-ci.txt | 1 - setup.cfg | 1 - 4 files changed, 6 insertions(+), 14 deletions(-) create mode 100644 changes/2014.misc diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 38fa8f1e9ca..04f22b54151 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -49,7 +49,7 @@ For standard python *venv*: $ python3 -m venv venv $ . venv/bin/activate -For *virtualenvwrapper* (my choice): +For *virtualenvwrapper*: .. code-block:: shell @@ -65,23 +65,16 @@ After that please install libraries required for development: $ pip install -r requirements-dev.txt -We also recommend to install ipdb_ but it's on your own: - -.. code-block:: shell - - $ pip install ipdb - .. note:: - If you plan to use ``ipdb`` within the test suite, execute: + If you plan to use ``pdb`` or ``ipdb`` within the test suite, execute: .. code-block:: shell - $ py.test tests -s -p no:timeout + $ py.test tests -s - command to run the tests with disabled timeout guard and output - capturing. + command to run the tests with disabled output capturing. -Congratulations, you are ready to run the test suite +Congratulations, you are ready to run the test suite! Run aiohttp test suite diff --git a/changes/2014.misc b/changes/2014.misc new file mode 100644 index 00000000000..b927e7b3a85 --- /dev/null +++ b/changes/2014.misc @@ -0,0 +1 @@ +Drop `pytest-timeout` plugin. diff --git a/requirements-ci.txt b/requirements-ci.txt index f7f27755934..c1167aa5351 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -15,7 +15,6 @@ sphinxcontrib-newsfeed==0.1.4 pytest==3.1.2 pytest-cov==2.5.1 pytest-mock==1.6.0 -pytest-timeout==1.2.0 gunicorn==19.7.1 pygments>=2.1 #aiodns # Enable from .travis.yml as required c-ares will not build on windows diff --git a/setup.cfg b/setup.cfg index 425a4336c45..6e6bf6b469f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,6 @@ max-line-length=79 [tool:pytest] testpaths = tests -timeout = 4 [isort] known_third_party=jinja2 From be88bf03b03224193b76081074963a0289b84d0e Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 26 Jun 2017 10:45:36 +0200 Subject: [PATCH 019/167] added a built with page in the docs (#2017) --- docs/built_with.rst | 20 ++++++++++++++++++++ docs/index.rst | 1 + docs/third_party.rst | 2 ++ 3 files changed, 23 insertions(+) create mode 100644 docs/built_with.rst diff --git a/docs/built_with.rst b/docs/built_with.rst new file mode 100644 index 00000000000..1a458f4c6a3 --- /dev/null +++ b/docs/built_with.rst @@ -0,0 +1,20 @@ +Built with aiohttp +================== + +aiohttp is used to build useful libraries built on top of it, +and there's a page dedicated to list them: :ref:`aiohttp-3rd-party`. + +There are also projects that leverage the power of aiohttp to +provide end-user tools, like command lines or softwares with +full user interfaces. + +This page aims to list those projects. If you are using aiohttp +in your software and if it's playing a central role, you +can add it here in this list. + +You can also add a **Built with aiohttp** link somewhere in your +project, pointing to ``_. + + +- `Molotov `_ Load testing tool. + diff --git a/docs/index.rst b/docs/index.rst index 66a6d82fc4a..913330c4e42 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -209,6 +209,7 @@ Contents deployment faq third_party + built_with essays contributing changes diff --git a/docs/third_party.rst b/docs/third_party.rst index b27d7e0ee60..8e47d00f321 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -1,3 +1,5 @@ +.. _aiohttp-3rd-party: + Third-Party libraries ===================== From 124ed0da12b7524f12aa142a8056171993aafa8d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 11:50:43 +0300 Subject: [PATCH 020/167] Increase timeout trying to make a test more reliable --- tests/test_client_functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index c1a550dd699..399705c3579 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -660,7 +660,7 @@ def handler(request): app.router.add_route('GET', '/', handler) client = yield from test_client(app) - resp = yield from client.get('/', timeout=0.05) + resp = yield from client.get('/', timeout=0.1) yield from fut with pytest.raises(asyncio.TimeoutError): From b8a2626a67e62b0035697de2a47461b9e7d17de6 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 26 Jun 2017 10:51:05 +0200 Subject: [PATCH 021/167] Update multidict from 3.0.0 to 3.1.0 (#2016) --- requirements-ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index c1167aa5351..d8b2aa20f5d 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -8,7 +8,7 @@ cython==0.25.2 chardet==3.0.4 isort==4.2.15 tox==2.7.0 -multidict==3.0.0 +multidict==3.1.0 async-timeout==1.2.1 sphinxcontrib-asyncio==0.2.0 sphinxcontrib-newsfeed==0.1.4 From 63b6c0ed0a5576e821ae5de017a37126debeb269 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 11:57:33 +0300 Subject: [PATCH 022/167] Doc fixes --- docs/web.rst | 2 +- docs/web_reference.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/web.rst b/docs/web.rst index 3f8cc2223bc..5845a82d79d 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -193,7 +193,7 @@ You can also specify a custom regex in the form ``{identifier:regex}``:: .. note:: Regex should match against *percent encoded* URL - (``request.rel_url_raw_path``). E.g. *space character* is encoded + (``request.raw_path``). E.g. *space character* is encoded as ``%20``. According to diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 7bac2644b96..943a1334bf1 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -144,7 +144,8 @@ and :ref:`aiohttp-web-signals` handlers. .. attribute:: raw_path The URL including raw *PATH INFO* without the host or scheme. - Warning, the path may be quoted and may contains non valid URL characters, e.g. + Warning, the path may be quoted and may contains non valid URL + characters, e.g. ``/my%2Fpath%7Cwith%21some%25strange%24characters``. For unquoted version please take a look on :attr:`path`. From 9dc842322e6bf7fe711a17ad00a8c40c22b99c44 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 12:55:39 +0300 Subject: [PATCH 023/167] Fix #1811: Add a note about possible performance dagradation in await resp.text() --- aiohttp/client_reqrep.py | 2 +- changes/1811.doc | 3 + docs/client_reference.rst | 150 ++++++++++++++++++++++++++------------ 3 files changed, 108 insertions(+), 47 deletions(-) create mode 100644 changes/1811.doc diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index ee810cf4564..3fa52de8d52 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -27,7 +27,7 @@ import chardet -__all__ = ('ClientRequest', 'ClientResponse') +__all__ = ('ClientRequest', 'ClientResponse', 'RequestInfo') RequestInfo = collections.namedtuple( diff --git a/changes/1811.doc b/changes/1811.doc new file mode 100644 index 00000000000..28251039930 --- /dev/null +++ b/changes/1811.doc @@ -0,0 +1,3 @@ +Add a note about possible performance degradation in `await +resp.text()` if charset was not provided by `Content-Type` HTTP +header. Pass explicit encoding to solve it. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 37478d67163..bddaf09cdfe 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -102,15 +102,20 @@ The client session supports the context manager protocol for self closing. One example is not processing cookies at all when working in proxy mode. - If no cookie processing is needed, a :class:`aiohttp.helpers.DummyCookieJar` - instance can be provided. + If no cookie processing is needed, a + :class:`aiohttp.helpers.DummyCookieJar` instance can be + provided. .. versionadded:: 0.22 - :param callable json_serialize: Json `serializer` function. (:func:`json.dumps` by default) + :param callable json_serialize: Json *serializer* callable. - :param bool raise_for_status: Automatically call `raise_for_status()` for each response. - (default is False) + By default :func:`json.dumps` function. + + :param bool raise_for_status: + + Automatically call :meth:`ClientResponse.raise_for_status()` for + each response, ``False`` by default. .. versionadded:: 2.0 @@ -153,10 +158,12 @@ The client session supports the context manager protocol for self closing. .. attribute:: requote_redirect_url aiohttp re quote's redirect urls by default, but some servers - require exact url from location header. to disable `re-quote` system - set `requote_redirect_url` to `False`. + require exact url from location header. to disable *re-quote* system + set :attr:`requote_redirect_url` attribute to ``False``. + + .. versionadded:: 2.1 - .. note:: this parameter affects all subsequent requests. + .. note:: This parameter affects all subsequent requests. .. attribute:: loop @@ -198,8 +205,9 @@ The client session supports the context manager protocol for self closing. :param data: Dictionary, bytes, or file-like object to send in the body of the request (optional) - :param json: Any json compatible python object (optional). `json` and `data` - parameters could not be used at the same time. + :param json: Any json compatible python object + (optional). *json* and *data* parameters could not + be used at the same time. :param dict headers: HTTP Headers to send with the request (optional) @@ -226,7 +234,8 @@ The client session supports the context manager protocol for self closing. with a *Content-Encoding* and *Content-Length* headers. ``None`` by default (optional). - :param int chunked: Enable chunked transfer encoding. It is up to the developer + :param int chunked: Enable chunked transfer encoding. + It is up to the developer to decide how to chunk data streams. If chunking is enabled, aiohttp encodes the provided chunks in the "Transfer-encoding: chunked" format. If *chunked* is set, then the *Transfer-encoding* and *content-length* @@ -244,9 +253,11 @@ The client session supports the context manager protocol for self closing. :param aiohttp.BasicAuth proxy_auth: an object that represents proxy HTTP Basic Authorization (optional) - :param int timeout: override the session's timeout (``read_timeout``) for IO operations. + :param int timeout: override the session's timeout + (``read_timeout``) for IO operations. - :return ClientResponse: a :class:`client response ` object. + :return ClientResponse: a :class:`client response ` + object. .. versionadded:: 1.0 @@ -431,24 +442,27 @@ The client session supports the context manager protocol for self closing. :param tuple protocols: Websocket protocols - :param float timeout: Timeout for websocket to close. 10 seconds by default + :param float timeout: Timeout for websocket to close. ``10`` seconds + by default - :param float receive_timeout: Timeout for websocket to receive complete message. - None(unlimited) seconds by default + :param float receive_timeout: Timeout for websocket to receive + complete message. ``None`` (unlimited) + seconds by default :param aiohttp.BasicAuth auth: an object that represents HTTP Basic Authorization (optional) :param bool autoclose: Automatically close websocket connection on close - message from server. If `autoclose` is False + message from server. If *autoclose* is False them close procedure has to be handled manually - :param bool autoping: automatically send `pong` on `ping` + :param bool autoping: automatically send *pong* on *ping* message from server - :param float heartbeat: Send `ping` message every `heartbeat` seconds - and wait `pong` response, if `pong` response is not received - then close connection. + :param float heartbeat: Send *ping* message every *heartbeat* + seconds and wait *pong* response, if + *pong* response is not received then + close connection. :param str origin: Origin header to send to server @@ -526,7 +540,7 @@ certification chaining. :param data: Dictionary, bytes, or file-like object to send in the body of the request (optional) - :param json: Any json compatible python object (optional). `json` and `data` + :param json: Any json compatible python object (optional). *json* and *data* parameters could not be used at the same time. :param dict headers: HTTP Headers to send with the request (optional) @@ -598,8 +612,8 @@ There are standard connectors: All connector classes should be derived from :class:`BaseConnector`. -By default all *connectors* support *keep-alive connections* (behavior is controlled by -*force_close* constructor's parameter). +By default all *connectors* support *keep-alive connections* (behavior +is controlled by *force_close* constructor's parameter). BaseConnector @@ -973,8 +987,9 @@ Response object Reading from the stream may raise :exc:`aiohttp.ClientPayloadError` if the response object is - closed before response receives all data or in case if any transfer encoding - related errors like mis-formed chunked encoding of broken compression data. + closed before response receives all data or in case if any + transfer encoding related errors like mis-formed chunked + encoding of broken compression data. .. attribute:: cookies @@ -1037,13 +1052,16 @@ Response object .. comethod:: release() - It is not required to call `release` on the response object. When the - client fully receives the payload, the underlying connection automatically - returns back to pool. If the payload is not fully read, the connection is closed + It is not required to call `release` on the response + object. When the client fully receives the payload, the + underlying connection automatically returns back to pool. If the + payload is not fully read, the connection is closed .. method:: raise_for_status() - Raise an :exc:`aiohttp.ClientResponseError` if the response status is 400 or higher. + Raise an :exc:`aiohttp.ClientResponseError` if the response + status is 400 or higher. + Do nothing for success responses (less than 400). .. comethod:: text(encoding=None) @@ -1052,7 +1070,10 @@ Response object specified *encoding* parameter. If *encoding* is ``None`` content encoding is autocalculated - using :term:`cchardet` or :term:`chardet` as fallback if + using ``Content-Type`` HTTP header and *chardet* tool if the + header is not provided by server. + + :term:`cchardet` is used with fallback to :term:`chardet` if *cchardet* is not available. Close underlying connection if data reading gets an error, @@ -1064,7 +1085,19 @@ Response object :return str: decoded *BODY* - .. comethod:: json(encoding=None, loads=json.loads, content_type='application/json') + .. note:: + + If response has no ``charset`` info in ``Content-Type`` HTTP + header :term:`cchardet` / :term:`chardet` is used for content + encoding autodetection. + + It may hurt performance. If page encoding is known passing + explicit *encoding* parameter might help:: + + await resp.text('ISO-8859-1') + + .. comethod:: json(*, encoding=None, loads=json.loads, \ + content_type='application/json') Read response's body as *JSON*, return :class:`dict` using specified *encoding* and *loader*. @@ -1092,10 +1125,10 @@ Response object :return: *BODY* as *JSON* data parsed by *loads* parameter or ``None`` if *BODY* is empty or contains white-spaces only. - .. attribute:: request_info + .. attribute:: request_info A namedtuple with request URL and headers from :class:`ClientRequest` - object. + object, :class:`aiohttp.RequestInfo` instance. ClientWebSocketResponse @@ -1240,6 +1273,27 @@ Utilities --------- +RequestInfo +^^^^^^^^^^^ + +.. class:: RequestInfo() + + A namedtuple with request URL and headers from :class:`ClientRequest` + object, available as :attr:`ClientResponse.request_info` attribute. + + .. attribute:: url + + Requested *url*, :class:`yarl.URL` instance. + + .. attribute:: method + + Request HTTP method like ``'GET'`` or ``'POST'``, :class:`str`. + + .. attribute:: headers + + HTTP headers for request, :class:`multidict.CIMultiDict` instance. + + BasicAuth ^^^^^^^^^ @@ -1249,7 +1303,7 @@ BasicAuth :param str login: login :param str password: password - :param str encoding: encoding (`'latin1'` by default) + :param str encoding: encoding (``'latin1'`` by default) Should be used for specifying authorization data in client API, @@ -1351,24 +1405,28 @@ CookieJar Client exceptions ^^^^^^^^^^^^^^^^^ -Exception hierarchy has been significantly modified in version 2.0. aiohttp defines only -exceptions that covers connection handling and server response misbehaviors. -For developer specific mistakes, aiohttp uses python standard exceptions -like `ValueError` or `TypeError`. +Exception hierarchy has been significantly modified in version +2.0. aiohttp defines only exceptions that covers connection handling +and server response misbehaviors. For developer specific mistakes, +aiohttp uses python standard exceptions like :exc:`ValueError` or +:exc:`TypeError`. -Reading a response content may raise a :exc:`ClientPayloadError` exception. This exception -indicates errors specific to the payload encoding. Such as invalid compressed data, -malformed chunked-encoded chunks or not enough data that satisfy the content-length header. +Reading a response content may raise a :exc:`ClientPayloadError` +exception. This exception indicates errors specific to the payload +encoding. Such as invalid compressed data, malformed chunked-encoded +chunks or not enough data that satisfy the content-length header. -All exceptions are available as attributes in `aiohttp` module. +All exceptions are available as members of *aiohttp* module. Hierarchy of exceptions: -* `aiohttp.ClientError` - Base class for all client specific exceptions +* :exc:`aiohttp.ClientError` - Base class for all client specific exceptions - - `aiohttp.ClientResponseError` - exceptions that could happen after we get response from server. + - :exc:`aiohttp.ClientResponseError` - exceptions that could happen after + we get response from server. - `request_info` - Instance of `RequestInfo` object, contains information about request. + `request_info` - Instance of `RequestInfo` object, contains + information about request. `history` - History from `ClientResponse` object, if available, else empty tuple. From 9ff2e40ecf3d1db14eda5bb861af401e9915bea9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 14:09:13 +0300 Subject: [PATCH 024/167] Fix #2009: Use ClientRedirectError for redirection problems --- aiohttp/client.py | 13 ++++++------- aiohttp/client_exceptions.py | 14 +++++++++++++- aiohttp/http_exceptions.py | 4 ---- changes/2009.feature | 4 ++-- docs/client_reference.rst | 3 +++ tests/test_client_functional.py | 11 +++-------- 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 0afafddcac6..de1310baeb5 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -15,7 +15,8 @@ from . import connector as connector_mod from . import client_exceptions, client_reqrep, hdrs, http, payload from .client_exceptions import * # noqa -from .client_exceptions import (ClientError, ClientOSError, ServerTimeoutError, +from .client_exceptions import (ClientError, ClientOSError, + ClientRedirectError, ServerTimeoutError, WSServerHandshakeError) from .client_reqrep import * # noqa from .client_reqrep import ClientRequest, ClientResponse @@ -26,7 +27,6 @@ from .helpers import (PY_35, CeilTimeout, TimeoutHandle, deprecated_noop, sentinel) from .http import WS_KEY, WebSocketReader, WebSocketWriter -from .http_exceptions import RedirectURLError from .streams import FlowControlDataQueue @@ -275,11 +275,10 @@ def _request(self, method, url, *, r_url = (resp.headers.get(hdrs.LOCATION) or resp.headers.get(hdrs.URI)) if r_url is None: - raise RedirectURLError( - "{0.method} {0.url} returns " - "a redirect [{0.status}] status " - "but response lacks a Location " - "or URI HTTP header".format(resp)) + raise ClientRedirectError( + resp.request_info, + history, + resp.status) r_url = URL( r_url, encoded=not self.requote_redirect_url) diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 98f6a764551..0e9e682ac96 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -12,7 +12,7 @@ 'ServerConnectionError', 'ServerTimeoutError', 'ServerDisconnectedError', 'ServerFingerprintMismatch', - 'ClientResponseError', 'ClientPayloadError', + 'ClientResponseError', 'ClientRedirectError', 'ClientPayloadError', 'ClientHttpProxyError', 'WSServerHandshakeError') @@ -37,6 +37,18 @@ def __init__(self, request_info, history, *, super().__init__("%s, message='%s'" % (code, message)) +class ClientRedirectError(ClientResponseError): + """Redirection error. + + Response is a redirect but Location or URI HTTP headers are + missing + + """ + def __init__(self, request_info, history, code): + super().__init__(request_info, history, code=code, + message="Response has no Location or URI header") + + class ClientPayloadError(ClientError): """Response payload error.""" diff --git a/aiohttp/http_exceptions.py b/aiohttp/http_exceptions.py index de42e2d19c3..dc2d1095f93 100644 --- a/aiohttp/http_exceptions.py +++ b/aiohttp/http_exceptions.py @@ -57,10 +57,6 @@ class ContentLengthError(PayloadEncodingError): """Not enough data for satisfy content length header.""" -class RedirectURLError(BadHttpMessage): - """A redirect response lacks a Location or URI HTTP header""" - - class LineTooLong(BadHttpMessage): def __init__(self, line, limit='Unknown'): diff --git a/changes/2009.feature b/changes/2009.feature index eb4dd60077a..c9b57d9445c 100644 --- a/changes/2009.feature +++ b/changes/2009.feature @@ -1,3 +1,3 @@ -`RedirectURLError` is raised instead of RuntimeError +`ClientRedirectLError` is raised instead of RuntimeError -Raise `RedirectURLError` if a redirect response has no Location or URI HTTP header. \ No newline at end of file +Raise `ClientRedirectError` if a redirect response has no Location or URI HTTP header. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index bddaf09cdfe..23f570ffedc 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1430,6 +1430,9 @@ Hierarchy of exceptions: `history` - History from `ClientResponse` object, if available, else empty tuple. + - `aiohttp.ClientRedirectError` - Response is a redirect but + ``Location`` or ``URI`` headers are missing. + - `aiohttp.WSServerHandshakeError` - web socket server response error - `aiohttp.ClientHttpProxyError` - proxy response diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 399705c3579..fe3f95d04b3 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -12,10 +12,8 @@ from multidict import MultiDict import aiohttp -from aiohttp import hdrs, web -from aiohttp.client import ServerFingerprintMismatch +from aiohttp import ClientRedirectError, ServerFingerprintMismatch, hdrs, web from aiohttp.helpers import create_future -from aiohttp.http_exceptions import RedirectURLError from aiohttp.multipart import MultipartWriter @@ -2107,12 +2105,9 @@ def handler_redirect(request): app.router.add_route('GET', '/redirect', handler_redirect) client = yield from test_client(app) - with pytest.raises(RedirectURLError) as ctx: + with pytest.raises(ClientRedirectError) as ctx: yield from client.get('/redirect') - expected_msg = ('GET http://127.0.0.1:{}/redirect returns ' - 'a redirect [301] status but response lacks ' - 'a Location or URI HTTP header' - .format(client.port)) + expected_msg = 'Response has no Location or URI header' assert str(ctx.value.message) == expected_msg From 99d78299c8ff0d9f81e5afb7947db01cbde8b319 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 14:47:58 +0300 Subject: [PATCH 025/167] #2013: Move github templates into .github folder --- ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE.md | 0 PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE.md (100%) rename PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md (100%) diff --git a/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md similarity index 100% rename from ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE.md diff --git a/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE.md From 5cb3de05402936f14c0c9f8808b6f3186dafba8d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 14:51:06 +0300 Subject: [PATCH 026/167] #2013: Move build.cmd into tools --- appveyor.yml | 10 +++++----- build.cmd => tools/build.cmd | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename build.cmd => tools/build.cmd (100%) diff --git a/appveyor.yml b/appveyor.yml index d7e1ab0865c..a5e1197e019 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,17 +13,17 @@ environment: - PYTHON: "C:\\Python36-x64" install: - - "build.cmd %PYTHON%\\python.exe -m pip install wheel" - - "build.cmd %PYTHON%\\python.exe -m pip install twine" - - "build.cmd %PYTHON%\\python.exe -m pip install -r requirements-ci.txt" + - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" + - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine" + - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements-ci.txt" build: false test_script: - - "build.cmd %PYTHON%\\python.exe setup.py test" + - "tools/build.cmd %PYTHON%\\python.exe setup.py test" after_test: - - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + - "tools/build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" artifacts: - path: dist\* diff --git a/build.cmd b/tools/build.cmd similarity index 100% rename from build.cmd rename to tools/build.cmd From f749f96498279fb7604471d171b5fc3e19181cba Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 14:55:58 +0300 Subject: [PATCH 027/167] #2013: move wheels build scripts into tools --- .travis.yml | 2 +- build-wheels.sh => tools/build-wheels.sh | 0 run_docker.sh => tools/run_docker.sh | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename build-wheels.sh => tools/build-wheels.sh (100%) rename run_docker.sh => tools/run_docker.sh (94%) diff --git a/.travis.yml b/.travis.yml index 1b45d4ac3f1..ed6630223ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ script: after_success: - codecov - - ./run_docker.sh + - ./tools/run_docker.sh deploy: provider: pypi diff --git a/build-wheels.sh b/tools/build-wheels.sh similarity index 100% rename from build-wheels.sh rename to tools/build-wheels.sh diff --git a/run_docker.sh b/tools/run_docker.sh similarity index 94% rename from run_docker.sh rename to tools/run_docker.sh index 30d3e3cadff..056eb51d583 100755 --- a/run_docker.sh +++ b/tools/run_docker.sh @@ -1,7 +1,7 @@ if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENSIONS] ;then echo "x86_64" docker pull quay.io/pypa/manylinux1_x86_64 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/build-wheels.sh + docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/tools/build-wheels.sh echo "Dist folder content is:" for f in dist/aiohttp*manylinux1_x86_64.whl do @@ -13,7 +13,7 @@ if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENS echo "i686" docker pull quay.io/pypa/manylinux1_i686 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/build-wheels.sh + docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/tools/build-wheels.sh echo "Dist folder content is:" for f in dist/aiohttp*manylinux1_i686.whl do From fefb85b41f0cbf29c18a8177e9b3563be890e330 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:01:23 +0300 Subject: [PATCH 028/167] #2013: Move requirements files into requirements folder --- .travis.yml | 2 +- CONTRIBUTING.rst | 2 +- Makefile | 6 +++--- appveyor.yml | 2 +- requirements-ci.txt => requirements/ci.txt | 0 requirements-dev.txt => requirements/dev.txt | 2 +- requirements-wheel.txt => requirements/wheel.txt | 0 tools/build-wheels.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename requirements-ci.txt => requirements/ci.txt (100%) rename requirements-dev.txt => requirements/dev.txt (77%) rename requirements-wheel.txt => requirements/wheel.txt (100%) diff --git a/.travis.yml b/.travis.yml index ed6630223ce..4aba6a9a40d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ install: - pip install --upgrade pip wheel - pip install --upgrade setuptools - pip install --upgrade setuptools-git - - pip install -r requirements-ci.txt + - pip install -r requirements/ci.txt - pip install aiodns - pip install codecov - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 04f22b54151..ee612235ceb 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -63,7 +63,7 @@ After that please install libraries required for development: .. code-block:: shell - $ pip install -r requirements-dev.txt + $ pip install -r requirements/dev.txt .. note:: If you plan to use ``pdb`` or ``ipdb`` within the test suite, execute: diff --git a/Makefile b/Makefile index 6da664c41ec..1867d66fede 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ all: test -.install-deps: requirements-dev.txt - @pip install -U -r requirements-dev.txt +.install-deps: requirements/dev.txt + @pip install -U -r requirements/dev.txt @touch .install-deps isort: @@ -102,6 +102,6 @@ doc-spelling: install: @pip install -U pip - @pip install -Ur requirements-dev.txt + @pip install -Ur requirements/dev.txt .PHONY: all build flake test vtest cov clean doc diff --git a/appveyor.yml b/appveyor.yml index a5e1197e019..f6931d03da4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,7 @@ environment: install: - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine" - - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements-ci.txt" + - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt" build: false diff --git a/requirements-ci.txt b/requirements/ci.txt similarity index 100% rename from requirements-ci.txt rename to requirements/ci.txt diff --git a/requirements-dev.txt b/requirements/dev.txt similarity index 77% rename from requirements-dev.txt rename to requirements/dev.txt index 6b7ec387ccd..3cc1cf2ea78 100644 --- a/requirements-dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ --r requirements-ci.txt +-r requirements/ci.txt ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 diff --git a/requirements-wheel.txt b/requirements/wheel.txt similarity index 100% rename from requirements-wheel.txt rename to requirements/wheel.txt diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 8033defed1c..4c2f2724daa 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -3,7 +3,7 @@ PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do - /opt/python/${PYTHON}/bin/pip install -r /io/requirements-wheel.txt + /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/ done From 53c58e5493d2f78d061b6a5f30745409b15e9a5e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:03:06 +0300 Subject: [PATCH 029/167] #2013: fix requirement import --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 3cc1cf2ea78..a92b636d73e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ --r requirements/ci.txt +-r ci.txt ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 From 6aea05b15a72f1afc8865776307251456a608a1b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:05:19 +0300 Subject: [PATCH 030/167] Update changes --- changes/2013.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/2013.misc diff --git a/changes/2013.misc b/changes/2013.misc new file mode 100644 index 00000000000..e9d8fc7e620 --- /dev/null +++ b/changes/2013.misc @@ -0,0 +1 @@ +Cleanup root folder. From 25ab86d9e9da53291ca09d3c5fb3e10b94771608 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:13:10 +0300 Subject: [PATCH 031/167] Fix #2018: Drop disqus widget from docs --- changes/2018.doc | 1 + docs/abc.rst | 4 ---- docs/api.rst | 4 ---- docs/changes.rst | 4 ---- docs/client.rst | 4 ---- docs/client_reference.rst | 4 ---- docs/conf.py | 4 ---- docs/contributing.rst | 4 ---- docs/deployment.rst | 3 --- docs/faq.rst | 4 ---- docs/glossary.rst | 5 ----- docs/index.rst | 4 ---- docs/logging.rst | 4 ---- docs/multipart.rst | 4 ---- docs/new_router.rst | 4 ---- docs/streams.rst | 4 ---- docs/testing.rst | 4 ---- docs/tutorial.rst | 3 --- docs/web.rst | 4 ---- docs/web_lowlevel.rst | 4 ---- docs/web_reference.rst | 4 ---- requirements/ci.txt | 1 - 22 files changed, 1 insertion(+), 80 deletions(-) create mode 100644 changes/2018.doc diff --git a/changes/2018.doc b/changes/2018.doc new file mode 100644 index 00000000000..2f5fd1e1eeb --- /dev/null +++ b/changes/2018.doc @@ -0,0 +1 @@ +Drop disqus widget from documentation pages. diff --git a/docs/abc.rst b/docs/abc.rst index 7caae117d9b..4faf7e3f2c4 100644 --- a/docs/abc.rst +++ b/docs/abc.rst @@ -146,7 +146,3 @@ Abstract Cookie Jar :return: :class:`http.cookies.SimpleCookie` with filtered cookies for given URL. - - -.. disqus:: - :title: aiohttp abstact base classes diff --git a/docs/api.rst b/docs/api.rst index 20e89398a50..3ba06be46a7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -198,7 +198,3 @@ aiohttp.signals module :members: :undoc-members: :show-inheritance: - - -.. disqus:: - :title: aiohttp helpers api diff --git a/docs/changes.rst b/docs/changes.rst index 005bd1fdc38..0ecf1d76af8 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -3,7 +3,3 @@ .. include:: ../CHANGES.rst .. include:: ../HISTORY.rst - - -.. disqus:: - :title: aiohttp changelog diff --git a/docs/client.rst b/docs/client.rst index c99c1cb4e34..76074bfd0b9 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -739,7 +739,3 @@ reading procedures:: Timeout is cumulative time, it includes all operations like sending request, redirects, response parsing, consuming response, etc. - - -.. disqus:: - :title: aiohttp client usage diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 23f570ffedc..e1464a077d1 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1458,7 +1458,3 @@ Hierarchy of exceptions: - `aiohttp.ClientPayloadError` - This exception can only be raised while reading the response payload if one of these errors occurs: invalid compression, malformed chunked encoding or not enough data that satisfy content-length header. - - -.. disqus:: - :title: aiohttp client reference diff --git a/docs/conf.py b/docs/conf.py index ed5da4b166d..67cbd10ea0f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,6 @@ 'sphinx.ext.intersphinx', 'alabaster', 'sphinxcontrib.asyncio', - 'sphinxcontrib.newsfeed', ] @@ -326,6 +325,3 @@ # If true, do not generate a @detailmenu in the "Top" node's menu. # texinfo_no_detailmenu = False - - -disqus_shortname = 'aiohttp' diff --git a/docs/contributing.rst b/docs/contributing.rst index 2f4d72b12c4..16acdbc0bdc 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1,7 +1,3 @@ .. _aiohttp-contributing: .. include:: ../CONTRIBUTING.rst - - -.. disqus:: - :title: instructions for aiohttp contributors diff --git a/docs/deployment.rst b/docs/deployment.rst index b052cc90da5..5fd4b3de6c4 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -306,6 +306,3 @@ By default aiohttp uses own defaults:: For more information please read :ref:`Format Specification for Access Log `. - -.. disqus:: - :title: aiohttp deployment with gunicorn diff --git a/docs/faq.rst b/docs/faq.rst index 8161180b6ca..c82ae0005a4 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,7 +361,3 @@ time consuming operation is very tricky matter. If you need global compression -- write own custom middleware. Or enable compression in NGINX (you are deploying aiohttp behind reverse proxy, is not it). - - -.. disqus:: - :title: aiohttp FAQ diff --git a/docs/glossary.rst b/docs/glossary.rst index 0288a848e8c..d95c20cb426 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -96,8 +96,3 @@ A library for operating with URL objects. https://pypi.python.org/pypi/yarl - - - -.. disqus:: - :title: aiohttp glossary diff --git a/docs/index.rst b/docs/index.rst index 913330c4e42..97f52a2dea9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -221,7 +221,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - - -.. disqus:: - :title: aiohttp documentation diff --git a/docs/logging.rst b/docs/logging.rst index b6ad5ed1f51..3a0686e56cd 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -117,7 +117,3 @@ To use different logger name please specify *logger* parameter .. _access_logformat: http://docs.gunicorn.org/en/stable/settings.html#access-log-format - - -.. disqus:: - :title: aiohttp logging diff --git a/docs/multipart.rst b/docs/multipart.rst index 552b97c9dc2..363bf565486 100644 --- a/docs/multipart.rst +++ b/docs/multipart.rst @@ -329,7 +329,3 @@ And this gives us a more cleaner solution:: .. _cgi.FieldStorage: https://docs.python.org/3.4/library/cgi.html .. _mimetypes: https://docs.python.org/3.4/library/mimetypes.html - - -.. disqus:: - :title: aiohttp suppport for multipart encoding diff --git a/docs/new_router.rst b/docs/new_router.rst index 7dd05384017..a88b20838aa 100644 --- a/docs/new_router.rst +++ b/docs/new_router.rst @@ -82,7 +82,3 @@ shortcut for:: ``app.router.register_route(...)`` is still supported, it creates :class:`aiohttp.web.ResourceAdapter` for every call (but it's deprecated now). - - -.. disqus:: - :title: aiohttp router refactoring notes diff --git a/docs/streams.rst b/docs/streams.rst index 1ad6dbb8b02..4e717a59e5c 100644 --- a/docs/streams.rst +++ b/docs/streams.rst @@ -160,7 +160,3 @@ Helpers .. comethod:: wait_eof() Wait for EOF. The given data may be accessible by upcoming read calls. - - -.. disqus:: - :title: aiohttp streaming api diff --git a/docs/testing.rst b/docs/testing.rst index 42910c3beef..221e65cf228 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -767,7 +767,3 @@ Utilities .. _pytest: http://pytest.org/latest/ .. _pytest-aiohttp: https://pypi.python.org/pypi/pytest-aiohttp - - -.. disqus:: - :title: aiohttp testing diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 87444df3849..bedcbc2b016 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -461,6 +461,3 @@ Registered overrides are trivial Jinja2 template renderers:: return response .. seealso:: :ref:`aiohttp-web-middlewares` - -.. disqus:: - :title: aiohttp server tutorial diff --git a/docs/web.rst b/docs/web.rst index 5845a82d79d..3e121c80cb1 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -1324,7 +1324,3 @@ Documentation and a complete tutorial of creating and running an app locally are available at aiohttp-devtools_. .. _aiohttp-devtools: https://github.com/aio-libs/aiohttp-devtools - - -.. disqus:: - :title: aiohttp server usage diff --git a/docs/web_lowlevel.rst b/docs/web_lowlevel.rst index 14960f000fc..df335195a2d 100644 --- a/docs/web_lowlevel.rst +++ b/docs/web_lowlevel.rst @@ -87,7 +87,3 @@ Web-Socket for every *path*. The example is very basic: it always return ``200 OK`` response, real life code should be much more complex. - - -.. disqus:: - :title: aiohttp.web low-level server diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 943a1334bf1..ef94e957961 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -2208,7 +2208,3 @@ Normalize path middleware If *merge_slashes* is True, merge multiple consecutive slashes in the path into one. - - -.. disqus:: - :title: aiohttp server reference diff --git a/requirements/ci.txt b/requirements/ci.txt index d8b2aa20f5d..ee547ec05fc 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -11,7 +11,6 @@ tox==2.7.0 multidict==3.1.0 async-timeout==1.2.1 sphinxcontrib-asyncio==0.2.0 -sphinxcontrib-newsfeed==0.1.4 pytest==3.1.2 pytest-cov==2.5.1 pytest-mock==1.6.0 From 74753ad7ba4f753779a16fb6a08c0ee2778ce3cb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:32:02 +0300 Subject: [PATCH 032/167] Polish CHANGES --- CHANGES.rst | 154 ++++++------ HISTORY.rst | 709 ++++++++++++++++++++++++++-------------------------- 2 files changed, 432 insertions(+), 431 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9b52edd5e92..99f0621f40e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,45 +17,45 @@ Changes 2.2.0 (2017-06-20) ================== -- Add doc for add_head, update doc for add_get. #1944 +- Add doc for add_head, update doc for add_get. (#1944) - Fixed consecutive calls for `Response.write_eof`. - Retain method attributes (e.g. :code:`__doc__`) when registering synchronous - handlers for resources. #1953 + handlers for resources. (#1953) -- Added signal TERM handling in `run_app` to gracefully exit #1932 +- Added signal TERM handling in `run_app` to gracefully exit (#1932) -- Fix websocket issues caused by frame fragmentation. #1962 +- Fix websocket issues caused by frame fragmentation. (#1962) - Raise RuntimeError is you try to set the Content Length and enable - chunked encoding at the same time #1941 + chunked encoding at the same time (#1941) - Small update for `unittest_run_loop` -- Use CIMultiDict for ClientRequest.skip_auto_headers #1970 +- Use CIMultiDict for ClientRequest.skip_auto_headers (#1970) - Fix wrong startup sequence: test server and `run_app()` are not raise - `DeprecationWarning` now #1947 + `DeprecationWarning` now (#1947) -- Make sure cleanup signal is sent if startup signal has been sent #1959 +- Make sure cleanup signal is sent if startup signal has been sent (#1959) -- Fixed server keep-alive handler, could cause 100% cpu utilization #1955 +- Fixed server keep-alive handler, could cause 100% cpu utilization (#1955) - Connection can be destroyed before response get processed if - `await aiohttp.request(..)` is used #1981 + `await aiohttp.request(..)` is used (#1981) -- MultipartReader does not work with -OO #1969 +- MultipartReader does not work with -OO (#1969) -- Fixed `ClientPayloadError` with blank `Content-Encoding` header #1931 +- Fixed `ClientPayloadError` with blank `Content-Encoding` header (#1931) -- Support `deflate` encoding implemented in `httpbin.org/deflate` #1918 +- Support `deflate` encoding implemented in `httpbin.org/deflate` (#1918) -- Fix BadStatusLine caused by extra `CRLF` after `POST` data #1792 +- Fix BadStatusLine caused by extra `CRLF` after `POST` data (#1792) -- Keep a reference to `ClientSession` in response object #1985 +- Keep a reference to `ClientSession` in response object (#1985) -- Deprecate undocumented `app.on_loop_available` signal #1978 +- Deprecate undocumented `app.on_loop_available` signal (#1978) @@ -66,54 +66,54 @@ Changes https://github.com/PyO3/tokio - Write to transport ``\r\n`` before closing after keepalive timeout, - otherwise client can not detect socket disconnection. #1883 + otherwise client can not detect socket disconnection. (#1883) - Only call `loop.close` in `run_app` if the user did *not* supply a loop. Useful for allowing clients to specify their own cleanup before closing the asyncio loop if they wish to tightly control loop behavior -- Content disposition with semicolon in filename #917 +- Content disposition with semicolon in filename (#917) -- Added `request_info` to response object and `ClientResponseError`. #1733 +- Added `request_info` to response object and `ClientResponseError`. (#1733) -- Added `history` to `ClientResponseError`. #1741 +- Added `history` to `ClientResponseError`. (#1741) -- Allow to disable redirect url re-quoting #1474 +- Allow to disable redirect url re-quoting (#1474) -- Handle RuntimeError from transport #1790 +- Handle RuntimeError from transport (#1790) -- Dropped "%O" in access logger #1673 +- Dropped "%O" in access logger (#1673) - Added `args` and `kwargs` to `unittest_run_loop`. Useful with other - decorators, for example `@patch`. #1803 + decorators, for example `@patch`. (#1803) -- Added `iter_chunks` to response.content object. #1805 +- Added `iter_chunks` to response.content object. (#1805) - Avoid creating TimerContext when there is no timeout to allow - compatibility with Tornado. #1817 #1180 + compatibility with Tornado. (#1817) (#1180) - Add `proxy_from_env` to `ClientRequest` to read from environment - variables. #1791 + variables. (#1791) -- Add DummyCookieJar helper. #1830 +- Add DummyCookieJar helper. (#1830) -- Fix assertion errors in Python 3.4 from noop helper. #1847 +- Fix assertion errors in Python 3.4 from noop helper. (#1847) -- Do not unquote `+` in match_info values #1816 +- Do not unquote `+` in match_info values (#1816) - Use Forwarded, X-Forwarded-Scheme and X-Forwarded-Host for better scheme and - host resolution. #1134 + host resolution. (#1134) -- Fix sub-application middlewares resolution order #1853 +- Fix sub-application middlewares resolution order (#1853) -- Fix applications comparison #1866 +- Fix applications comparison (#1866) -- Fix static location in index when prefix is used #1662 +- Fix static location in index when prefix is used (#1662) -- Make test server more reliable #1896 +- Make test server more reliable (#1896) - Extend list of web exceptions, add HTTPUnprocessableEntity, - HTTPFailedDependency, HTTPInsufficientStorage status codes #1920 + HTTPFailedDependency, HTTPInsufficientStorage status codes (#1920) 2.0.7 (2017-04-12) @@ -121,57 +121,57 @@ Changes - Fix *pypi* distribution -- Fix exception description #1807 +- Fix exception description (#1807) -- Handle socket error in FileResponse #1773 +- Handle socket error in FileResponse (#1773) -- Cancel websocket heartbeat on close #1793 +- Cancel websocket heartbeat on close (#1793) 2.0.6 (2017-04-04) ================== -- Keeping blank values for `request.post()` and `multipart.form()` #1765 +- Keeping blank values for `request.post()` and `multipart.form()` (#1765) -- TypeError in data_received of ResponseHandler #1770 +- TypeError in data_received of ResponseHandler (#1770) - Fix ``web.run_app`` not to bind to default host-port pair if only socket is - passed #1786 + passed (#1786) 2.0.5 (2017-03-29) ================== -- Memory leak with aiohttp.request #1756 +- Memory leak with aiohttp.request (#1756) - Disable cleanup closed ssl transports by default. - Exception in request handling if the server responds before the body - is sent #1761 + is sent (#1761) 2.0.4 (2017-03-27) ================== -- Memory leak with aiohttp.request #1756 +- Memory leak with aiohttp.request (#1756) -- Encoding is always UTF-8 in POST data #1750 +- Encoding is always UTF-8 in POST data (#1750) -- Do not add "Content-Disposition" header by default #1755 +- Do not add "Content-Disposition" header by default (#1755) 2.0.3 (2017-03-24) ================== -- Call https website through proxy will cause error #1745 +- Call https website through proxy will cause error (#1745) -- Fix exception on multipart/form-data post if content-type is not set #1743 +- Fix exception on multipart/form-data post if content-type is not set (#1743) 2.0.2 (2017-03-21) ================== -- Fixed Application.on_loop_available signal #1739 +- Fixed Application.on_loop_available signal (#1739) - Remove debug code @@ -179,21 +179,21 @@ Changes 2.0.1 (2017-03-21) ================== -- Fix allow-head to include name on route #1737 +- Fix allow-head to include name on route (#1737) -- Fixed AttributeError in WebSocketResponse.can_prepare #1736 +- Fixed AttributeError in WebSocketResponse.can_prepare (#1736) 2.0.0 (2017-03-20) ================== -- Added `json` to `ClientSession.request()` method #1726 +- Added `json` to `ClientSession.request()` method (#1726) - Added session's `raise_for_status` parameter, automatically calls - raise_for_status() on any request. #1724 + raise_for_status() on any request. (#1724) - `response.json()` raises `ClientReponseError` exception if response's - content type does not match #1723 + content type does not match (#1723) - Cleanup timer and loop handle on any client exception. @@ -203,25 +203,25 @@ Changes `2.0.0rc1` (2017-03-15) ======================= -- Properly handle payload errors #1710 +- Properly handle payload errors (#1710) -- Added `ClientWebSocketResponse.get_extra_info()` #1717 +- Added `ClientWebSocketResponse.get_extra_info()` (#1717) - It is not possible to combine Transfer-Encoding and chunked parameter, - same for compress and Content-Encoding #1655 + same for compress and Content-Encoding (#1655) - Connector's `limit` parameter indicates total concurrent connections. - New `limit_per_host` added, indicates total connections per endpoint. #1601 + New `limit_per_host` added, indicates total connections per endpoint. (#1601) -- Use url's `raw_host` for name resolution #1685 +- Use url's `raw_host` for name resolution (#1685) -- Change `ClientResponse.url` to `yarl.URL` instance #1654 +- Change `ClientResponse.url` to `yarl.URL` instance (#1654) -- Add max_size parameter to web.Request reading methods #1133 +- Add max_size parameter to web.Request reading methods (#1133) -- Web Request.post() stores data in temp files #1469 +- Web Request.post() stores data in temp files (#1469) -- Add the `allow_head=True` keyword argument for `add_get` #1618 +- Add the `allow_head=True` keyword argument for `add_get` (#1618) - `run_app` and the Command Line Interface now support serving over Unix domain sockets for faster inter-process communication. @@ -230,45 +230,45 @@ Changes e.g. for socket-based activated applications, when binding of a socket is done by the parent process. -- Implementation for Trailer headers parser is broken #1619 +- Implementation for Trailer headers parser is broken (#1619) - Fix FileResponse to not fall on bad request (range out of file size) - Fix FileResponse to correct stream video to Chromes -- Deprecate public low-level api #1657 +- Deprecate public low-level api (#1657) - Deprecate `encoding` parameter for ClientSession.request() method -- Dropped aiohttp.wsgi #1108 +- Dropped aiohttp.wsgi (#1108) - Dropped `version` from ClientSession.request() method -- Dropped websocket version 76 support #1160 +- Dropped websocket version 76 support (#1160) -- Dropped: `aiohttp.protocol.HttpPrefixParser` #1590 +- Dropped: `aiohttp.protocol.HttpPrefixParser` (#1590) - Dropped: Servers response's `.started`, `.start()` and - `.can_start()` method #1591 + `.can_start()` method (#1591) - Dropped: Adding `sub app` via `app.router.add_subapp()` is deprecated - use `app.add_subapp()` instead #1592 + use `app.add_subapp()` instead (#1592) -- Dropped: `Application.finish()` and `Application.register_on_finish()` #1602 +- Dropped: `Application.finish()` and `Application.register_on_finish()` (#1602) - Dropped: `web.Request.GET` and `web.Request.POST` - Dropped: aiohttp.get(), aiohttp.options(), aiohttp.head(), aiohttp.post(), aiohttp.put(), aiohttp.patch(), aiohttp.delete(), and - aiohttp.ws_connect() #1593 + aiohttp.ws_connect() (#1593) -- Dropped: `aiohttp.web.WebSocketResponse.receive_msg()` #1605 +- Dropped: `aiohttp.web.WebSocketResponse.receive_msg()` (#1605) - Dropped: `ServerHttpProtocol.keep_alive_timeout` attribute and - `keep-alive`, `keep_alive_on`, `timeout`, `log` constructor parameters #1606 + `keep-alive`, `keep_alive_on`, `timeout`, `log` constructor parameters (#1606) - Dropped: `TCPConnector's`` `.resolve`, `.resolved_hosts`, `.clear_resolved_hosts()` attributes and `resolve` constructor - parameter #1607 + parameter (#1607) -- Dropped `ProxyConnector` #1609 +- Dropped `ProxyConnector` (#1609) diff --git a/HISTORY.rst b/HISTORY.rst index c2f5b6fcf47..76720596ea4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,7 +1,7 @@ 1.3.5 (2017-03-16) ================== -- Fixed None timeout support #1720 +- Fixed None timeout support (#1720) 1.3.4 (2017-03-14) @@ -15,29 +15,30 @@ - Fix file_sender to correct stream video to Chromes -- Fix NotImplementedError server exception #1703 +- Fix NotImplementedError server exception (#1703) -- Clearer error message for URL without a host name. #1691 +- Clearer error message for URL without a host name. (#1691) -- Silence deprecation warning in __repr__ #1690 +- Silence deprecation warning in __repr__ (#1690) -- IDN + HTTPS = `ssl.CertificateError` #1685 +- IDN + HTTPS = `ssl.CertificateError` (#1685) 1.3.3 (2017-02-19) ================== -- Fixed memory leak in time service #1656 +- Fixed memory leak in time service (#1656) 1.3.2 (2017-02-16) ================== -- Awaiting on WebSocketResponse.send_* does not work #1645 +- Awaiting on WebSocketResponse.send_* does not work (#1645) -- Fix multiple calls to client ws_connect when using a shared header dict #1643 +- Fix multiple calls to client ws_connect when using a shared header + dict (#1643) -- Make CookieJar.filter_cookies() accept plain string parameter. #1636 +- Make CookieJar.filter_cookies() accept plain string parameter. (#1636) 1.3.1 (2017-02-09) @@ -45,65 +46,65 @@ - Handle CLOSING in WebSocketResponse.__anext__ -- Fixed AttributeError 'drain' for server websocket handler #1613 +- Fixed AttributeError 'drain' for server websocket handler (#1613) 1.3.0 (2017-02-08) ================== - Multipart writer validates the data on append instead of on a - request send #920 + request send (#920) - Multipart reader accepts multipart messages with or without their epilogue - to consistently handle valid and legacy behaviors #1526 #1581 + to consistently handle valid and legacy behaviors (#1526) (#1581) - Separate read + connect + request timeouts # 1523 -- Do not swallow Upgrade header #1587 +- Do not swallow Upgrade header (#1587) -- Fix polls demo run application #1487 +- Fix polls demo run application (#1487) -- Ignore unknown 1XX status codes in client #1353 +- Ignore unknown 1XX status codes in client (#1353) -- Fix sub-Multipart messages missing their headers on serialization #1525 +- Fix sub-Multipart messages missing their headers on serialization (#1525) - Do not use readline when reading the content of a part - in the multipart reader #1535 + in the multipart reader (#1535) -- Add optional flag for quoting `FormData` fields #916 +- Add optional flag for quoting `FormData` fields (#916) -- 416 Range Not Satisfiable if requested range end > file size #1588 +- 416 Range Not Satisfiable if requested range end > file size (#1588) -- Having a `:` or `@` in a route does not work #1552 +- Having a `:` or `@` in a route does not work (#1552) - Added `receive_timeout` timeout for websocket to receive complete - message. #1325 + message. (#1325) - Added `heartbeat` parameter for websocket to automatically send - `ping` message. #1024 #777 + `ping` message. (#1024) (#777) -- Remove `web.Application` dependency from `web.UrlDispatcher` #1510 +- Remove `web.Application` dependency from `web.UrlDispatcher` (#1510) -- Accepting back-pressure from slow websocket clients #1367 +- Accepting back-pressure from slow websocket clients (#1367) -- Do not pause transport during set_parser stage #1211 +- Do not pause transport during set_parser stage (#1211) -- Lingering close does not terminate before timeout #1559 +- Lingering close does not terminate before timeout (#1559) -- `setsockopt` may raise `OSError` exception if socket is closed already #1595 +- `setsockopt` may raise `OSError` exception if socket is closed already (#1595) -- Lots of CancelledError when requests are interrupted #1565 +- Lots of CancelledError when requests are interrupted (#1565) - Allow users to specify what should happen to decoding errors - when calling a responses `text()` method #1542 + when calling a responses `text()` method (#1542) -- Back port std module `http.cookies` for python3.4.2 #1566 +- Back port std module `http.cookies` for python3.4.2 (#1566) -- Maintain url's fragment in client response #1314 +- Maintain url's fragment in client response (#1314) -- Allow concurrently close WebSocket connection #754 +- Allow concurrently close WebSocket connection (#754) -- Gzipped responses with empty body raises ContentEncodingError #609 +- Gzipped responses with empty body raises ContentEncodingError (#609) - Return 504 if request handle raises TimeoutError. @@ -113,25 +114,25 @@ message during client response release - Abort closed ssl client transports, broken servers can keep socket - open un-limit time #1568 + open un-limit time (#1568) - Log warning instead of `RuntimeError` is websocket connection is closed. - Deprecated: `aiohttp.protocol.HttpPrefixParser` - will be removed in 1.4 #1590 + will be removed in 1.4 (#1590) - Deprecated: Servers response's `.started`, `.start()` and - `.can_start()` method will be removed in 1.4 #1591 + `.can_start()` method will be removed in 1.4 (#1591) - Deprecated: Adding `sub app` via `app.router.add_subapp()` is deprecated - use `app.add_subapp()` instead, will be removed in 1.4 #1592 + use `app.add_subapp()` instead, will be removed in 1.4 (#1592) - Deprecated: aiohttp.get(), aiohttp.options(), aiohttp.head(), aiohttp.post(), aiohttp.put(), aiohttp.patch(), aiohttp.delete(), and aiohttp.ws_connect() - will be removed in 1.4 #1593 + will be removed in 1.4 (#1593) - Deprecated: `Application.finish()` and `Application.register_on_finish()` - will be removed in 1.4 #1602 + will be removed in 1.4 (#1602) 1.2.0 (2016-12-17) @@ -139,13 +140,13 @@ - Extract `BaseRequest` from `web.Request`, introduce `web.Server` (former `RequestHandlerFactory`), introduce new low-level web server - which is not coupled with `web.Application` and routing #1362 + which is not coupled with `web.Application` and routing (#1362) -- Make `TestServer.make_url` compatible with `yarl.URL` #1389 +- Make `TestServer.make_url` compatible with `yarl.URL` (#1389) -- Implement range requests for static files #1382 +- Implement range requests for static files (#1382) -- Support task attribute for StreamResponse #1410 +- Support task attribute for StreamResponse (#1410) - Drop `TestClient.app` property, use `TestClient.server.app` instead (BACKWARD INCOMPATIBLE) @@ -156,84 +157,84 @@ - `TestClient.server` property returns a test server instance, was `asyncio.AbstractServer` (BACKWARD INCOMPATIBLE) -- Follow gunicorn's signal semantics in `Gunicorn[UVLoop]WebWorker` #1201 +- Follow gunicorn's signal semantics in `Gunicorn[UVLoop]WebWorker` (#1201) - Call worker_int and worker_abort callbacks in - `Gunicorn[UVLoop]WebWorker` #1202 + `Gunicorn[UVLoop]WebWorker` (#1202) -- Has functional tests for client proxy #1218 +- Has functional tests for client proxy (#1218) -- Fix bugs with client proxy target path and proxy host with port #1413 +- Fix bugs with client proxy target path and proxy host with port (#1413) -- Fix bugs related to the use of unicode hostnames #1444 +- Fix bugs related to the use of unicode hostnames (#1444) -- Preserve cookie quoting/escaping #1453 +- Preserve cookie quoting/escaping (#1453) -- FileSender will send gzipped response if gzip version available #1426 +- FileSender will send gzipped response if gzip version available (#1426) - Don't override `Content-Length` header in `web.Response` if no body - was set #1400 + was set (#1400) -- Introduce `router.post_init()` for solving #1373 +- Introduce `router.post_init()` for solving (#1373) - Fix raise error in case of multiple calls of `TimeServive.stop()` -- Allow to raise web exceptions on router resolving stage #1460 +- Allow to raise web exceptions on router resolving stage (#1460) -- Add a warning for session creation outside of coroutine #1468 +- Add a warning for session creation outside of coroutine (#1468) - Avoid a race when application might start accepting incoming requests but startup signals are not processed yet e98e8c6 - Raise a `RuntimeError` when trying to change the status of the HTTP response - after the headers have been sent #1480 + after the headers have been sent (#1480) -- Fix bug with https proxy acquired cleanup #1340 +- Fix bug with https proxy acquired cleanup (#1340) -- Use UTF-8 as the default encoding for multipart text parts #1484 +- Use UTF-8 as the default encoding for multipart text parts (#1484) 1.1.6 (2016-11-28) ================== - Fix `BodyPartReader.read_chunk` bug about returns zero bytes before - `EOF` #1428 + `EOF` (#1428) 1.1.5 (2016-11-16) ================== -- Fix static file serving in fallback mode #1401 +- Fix static file serving in fallback mode (#1401) 1.1.4 (2016-11-14) ================== -- Make `TestServer.make_url` compatible with `yarl.URL` #1389 +- Make `TestServer.make_url` compatible with `yarl.URL` (#1389) - Generate informative exception on redirects from server which - does not provide redirection headers #1396 + does not provide redirection headers (#1396) 1.1.3 (2016-11-10) ================== -- Support *root* resources for sub-applications #1379 +- Support *root* resources for sub-applications (#1379) 1.1.2 (2016-11-08) ================== -- Allow starting variables with an underscore #1379 +- Allow starting variables with an underscore (#1379) -- Properly process UNIX sockets by gunicorn worker #1375 +- Properly process UNIX sockets by gunicorn worker (#1375) - Fix ordering for `FrozenList` -- Don't propagate pre and post signals to sub-application #1377 +- Don't propagate pre and post signals to sub-application (#1377) 1.1.1 (2016-11-04) ================== -- Fix documentation generation #1120 +- Fix documentation generation (#1120) 1.1.0 (2016-11-03) ================== @@ -241,23 +242,23 @@ - Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) - Use `yarl.URL` in client API. The change is 99% backward compatible - but `ClientResponse.url` is an `yarl.URL` instance now. #1217 + but `ClientResponse.url` is an `yarl.URL` instance now. (#1217) -- Close idle keep-alive connections on shutdown #1222 +- Close idle keep-alive connections on shutdown (#1222) -- Modify regex in AccessLogger to accept underscore and numbers #1225 +- Modify regex in AccessLogger to accept underscore and numbers (#1225) - Use `yarl.URL` in web server API. `web.Request.rel_url` and `web.Request.url` are added. URLs and templates are percent-encoded - now. #1224 + now. (#1224) -- Accept `yarl.URL` by server redirections #1278 +- Accept `yarl.URL` by server redirections (#1278) -- Return `yarl.URL` by `.make_url()` testing utility #1279 +- Return `yarl.URL` by `.make_url()` testing utility (#1279) -- Properly format IPv6 addresses by `aiohttp.web.run_app` #1139 +- Properly format IPv6 addresses by `aiohttp.web.run_app` (#1139) -- Use `yarl.URL` by server API #1288 +- Use `yarl.URL` by server API (#1288) * Introduce `resource.url_for()`, deprecate `resource.url()`. @@ -268,38 +269,38 @@ * Drop old-style routes: `Route`, `PlainRoute`, `DynamicRoute`, `StaticRoute`, `ResourceAdapter`. -- Revert `resp.url` back to `str`, introduce `resp.url_obj` #1292 +- Revert `resp.url` back to `str`, introduce `resp.url_obj` (#1292) -- Raise ValueError if BasicAuth login has a ":" character #1307 +- Raise ValueError if BasicAuth login has a ":" character (#1307) - Fix bug when ClientRequest send payload file with opened as - open('filename', 'r+b') #1306 + open('filename', 'r+b') (#1306) -- Enhancement to AccessLogger (pass *extra* dict) #1303 +- Enhancement to AccessLogger (pass *extra* dict) (#1303) -- Show more verbose message on import errors #1319 +- Show more verbose message on import errors (#1319) -- Added save and load functionality for `CookieJar` #1219 +- Added save and load functionality for `CookieJar` (#1219) -- Added option on `StaticRoute` to follow symlinks #1299 +- Added option on `StaticRoute` to follow symlinks (#1299) -- Force encoding of `application/json` content type to utf-8 #1339 +- Force encoding of `application/json` content type to utf-8 (#1339) -- Fix invalid invocations of `errors.LineTooLong` #1335 +- Fix invalid invocations of `errors.LineTooLong` (#1335) -- Websockets: Stop `async for` iteration when connection is closed #1144 +- Websockets: Stop `async for` iteration when connection is closed (#1144) -- Ensure TestClient HTTP methods return a context manager #1318 +- Ensure TestClient HTTP methods return a context manager (#1318) - Raise `ClientDisconnectedError` to `FlowControlStreamReader` read function - if `ClientSession` object is closed by client when reading data. #1323 + if `ClientSession` object is closed by client when reading data. (#1323) -- Document deployment without `Gunicorn` #1120 +- Document deployment without `Gunicorn` (#1120) - Add deprecation warning for MD5 and SHA1 digests when used for fingerprint - of site certs in TCPConnector. #1186 + of site certs in TCPConnector. (#1186) -- Implement sub-applications #1301 +- Implement sub-applications (#1301) - Don't inherit `web.Request` from `dict` but implement `MutableMapping` protocol. @@ -324,55 +325,55 @@ boost of your application -- a couple DB requests and business logic is still the main bottleneck. -- Boost performance by adding a custom time service #1350 +- Boost performance by adding a custom time service (#1350) - Extend `ClientResponse` with `content_type` and `charset` - properties like in `web.Request`. #1349 + properties like in `web.Request`. (#1349) -- Disable aiodns by default #559 +- Disable aiodns by default (#559) - Don't flap `tcp_cork` in client code, use TCP_NODELAY mode by default. -- Implement `web.Request.clone()` #1361 +- Implement `web.Request.clone()` (#1361) 1.0.5 (2016-10-11) ================== - Fix StreamReader._read_nowait to return all available - data up to the requested amount #1297 + data up to the requested amount (#1297) 1.0.4 (2016-09-22) ================== - Fix FlowControlStreamReader.read_nowait so that it checks - whether the transport is paused #1206 + whether the transport is paused (#1206) 1.0.2 (2016-09-22) ================== -- Make CookieJar compatible with 32-bit systems #1188 +- Make CookieJar compatible with 32-bit systems (#1188) -- Add missing `WSMsgType` to `web_ws.__all__`, see #1200 +- Add missing `WSMsgType` to `web_ws.__all__`, see (#1200) -- Fix `CookieJar` ctor when called with `loop=None` #1203 +- Fix `CookieJar` ctor when called with `loop=None` (#1203) -- Fix broken upper-casing in wsgi support #1197 +- Fix broken upper-casing in wsgi support (#1197) 1.0.1 (2016-09-16) ================== - Restore `aiohttp.web.MsgType` alias for `aiohttp.WSMsgType` for sake - of backward compatibility #1178 + of backward compatibility (#1178) - Tune alabaster schema. - Use `text/html` content type for displaying index pages by static file handler. -- Fix `AssertionError` in static file handling #1177 +- Fix `AssertionError` in static file handling (#1177) - Fix access log formats `%O` and `%b` for static file handling @@ -384,9 +385,9 @@ ================== - Change default size for client session's connection pool from - unlimited to 20 #977 + unlimited to 20 (#977) -- Add IE support for cookie deletion. #994 +- Add IE support for cookie deletion. (#994) - Remove deprecated `WebSocketResponse.wait_closed` method (BACKWARD INCOMPATIBLE) @@ -395,26 +396,26 @@ method (BACKWARD INCOMPATIBLE) - Avoid using of mutable CIMultiDict kw param in make_mocked_request - #997 + (#997) - Make WebSocketResponse.close a little bit faster by avoiding new task creating just for timeout measurement - Add `proxy` and `proxy_auth` params to `client.get()` and family, - deprecate `ProxyConnector` #998 + deprecate `ProxyConnector` (#998) - Add support for websocket send_json and receive_json, synchronize - server and client API for websockets #984 + server and client API for websockets (#984) - Implement router shourtcuts for most useful HTTP methods, use `app.router.add_get()`, `app.router.add_post()` etc. instead of - `app.router.add_route()` #986 + `app.router.add_route()` (#986) -- Support SSL connections for gunicorn worker #1003 +- Support SSL connections for gunicorn worker (#1003) - Move obsolete examples to legacy folder -- Switch to multidict 2.0 and title-cased strings #1015 +- Switch to multidict 2.0 and title-cased strings (#1015) - `{FOO}e` logger format is case-sensitive now @@ -430,9 +431,9 @@ - Remove deprecated decode param from resp.read(decode=True) -- Use 5min default client timeout #1028 +- Use 5min default client timeout (#1028) -- Relax HTTP method validation in UrlDispatcher #1037 +- Relax HTTP method validation in UrlDispatcher (#1037) - Pin minimal supported asyncio version to 3.4.2+ (`loop.is_close()` should be present) @@ -442,84 +443,84 @@ - Link header for 451 status code is mandatory -- Fix test_client fixture to allow multiple clients per test #1072 +- Fix test_client fixture to allow multiple clients per test (#1072) -- make_mocked_request now accepts dict as headers #1073 +- make_mocked_request now accepts dict as headers (#1073) - Add Python 3.5.2/3.6+ compatibility patch for async generator - protocol change #1082 + protocol change (#1082) -- Improvement test_client can accept instance object #1083 +- Improvement test_client can accept instance object (#1083) -- Simplify ServerHttpProtocol implementation #1060 +- Simplify ServerHttpProtocol implementation (#1060) - Add a flag for optional showing directory index for static file - handling #921 + handling (#921) -- Define `web.Application.on_startup()` signal handler #1103 +- Define `web.Application.on_startup()` signal handler (#1103) -- Drop ChunkedParser and LinesParser #1111 +- Drop ChunkedParser and LinesParser (#1111) -- Call `Application.startup` in GunicornWebWorker #1105 +- Call `Application.startup` in GunicornWebWorker (#1105) - Fix client handling hostnames with 63 bytes when a port is given in - the url #1044 + the url (#1044) -- Implement proxy support for ClientSession.ws_connect #1025 +- Implement proxy support for ClientSession.ws_connect (#1025) -- Return named tuple from WebSocketResponse.can_prepare #1016 +- Return named tuple from WebSocketResponse.can_prepare (#1016) -- Fix access_log_format in `GunicornWebWorker` #1117 +- Fix access_log_format in `GunicornWebWorker` (#1117) -- Setup Content-Type to application/octet-stream by default #1124 +- Setup Content-Type to application/octet-stream by default (#1124) - Deprecate debug parameter from app.make_handler(), use - `Application(debug=True)` instead #1121 + `Application(debug=True)` instead (#1121) -- Remove fragment string in request path #846 +- Remove fragment string in request path (#846) -- Use aiodns.DNSResolver.gethostbyname() if available #1136 +- Use aiodns.DNSResolver.gethostbyname() if available (#1136) -- Fix static file sending on uvloop when sendfile is available #1093 +- Fix static file sending on uvloop when sendfile is available (#1093) -- Make prettier urls if query is empty dict #1143 +- Make prettier urls if query is empty dict (#1143) -- Fix redirects for HEAD requests #1147 +- Fix redirects for HEAD requests (#1147) -- Default value for `StreamReader.read_nowait` is -1 from now #1150 +- Default value for `StreamReader.read_nowait` is -1 from now (#1150) - `aiohttp.StreamReader` is not inherited from `asyncio.StreamReader` from now - (BACKWARD INCOMPATIBLE) #1150 + (BACKWARD INCOMPATIBLE) (#1150) -- Streams documentation added #1150 +- Streams documentation added (#1150) -- Add `multipart` coroutine method for web Request object #1067 +- Add `multipart` coroutine method for web Request object (#1067) -- Publish ClientSession.loop property #1149 +- Publish ClientSession.loop property (#1149) -- Fix static file with spaces #1140 +- Fix static file with spaces (#1140) -- Fix piling up asyncio loop by cookie expiration callbacks #1061 +- Fix piling up asyncio loop by cookie expiration callbacks (#1061) - Drop `Timeout` class for sake of `async_timeout` external library. `aiohttp.Timeout` is an alias for `async_timeout.timeout` - `use_dns_cache` parameter of `aiohttp.TCPConnector` is `True` by - default (BACKWARD INCOMPATIBLE) #1152 + default (BACKWARD INCOMPATIBLE) (#1152) - `aiohttp.TCPConnector` uses asynchronous DNS resolver if available by - default (BACKWARD INCOMPATIBLE) #1152 + default (BACKWARD INCOMPATIBLE) (#1152) -- Conform to RFC3986 - do not include url fragments in client requests #1174 +- Conform to RFC3986 - do not include url fragments in client requests (#1174) -- Drop `ClientSession.cookies` (BACKWARD INCOMPATIBLE) #1173 +- Drop `ClientSession.cookies` (BACKWARD INCOMPATIBLE) (#1173) -- Refactor `AbstractCookieJar` public API (BACKWARD INCOMPATIBLE) #1173 +- Refactor `AbstractCookieJar` public API (BACKWARD INCOMPATIBLE) (#1173) - Fix clashing cookies with have the same name but belong to different - domains (BACKWARD INCOMPATIBLE) #1125 + domains (BACKWARD INCOMPATIBLE) (#1125) -- Support binary Content-Transfer-Encoding #1169 +- Support binary Content-Transfer-Encoding (#1169) 0.22.5 (08-02-2016) @@ -530,17 +531,17 @@ 0.22.3 (07-26-2016) =================== -- Do not filter cookies if unsafe flag provided #1005 +- Do not filter cookies if unsafe flag provided (#1005) 0.22.2 (07-23-2016) =================== -- Suppress CancelledError when Timeout raises TimeoutError #970 +- Suppress CancelledError when Timeout raises TimeoutError (#970) - Don't expose `aiohttp.__version__` -- Add unsafe parameter to CookieJar #968 +- Add unsafe parameter to CookieJar (#968) - Use unsafe cookie jar in test client tools @@ -551,88 +552,88 @@ =================== - Large cookie expiration/max-age does not break an event loop from now - (fixes #967) + (fixes (#967)) 0.22.0 (07-15-2016) =================== -- Fix bug in serving static directory #803 +- Fix bug in serving static directory (#803) -- Fix command line arg parsing #797 +- Fix command line arg parsing (#797) -- Fix a documentation chapter about cookie usage #790 +- Fix a documentation chapter about cookie usage (#790) -- Handle empty body with gzipped encoding #758 +- Handle empty body with gzipped encoding (#758) -- Support 451 Unavailable For Legal Reasons http status #697 +- Support 451 Unavailable For Legal Reasons http status (#697) -- Fix Cookie share example and few small typos in docs #817 +- Fix Cookie share example and few small typos in docs (#817) -- UrlDispatcher.add_route with partial coroutine handler #814 +- UrlDispatcher.add_route with partial coroutine handler (#814) -- Optional support for aiodns #728 +- Optional support for aiodns (#728) -- Add ServiceRestart and TryAgainLater websocket close codes #828 +- Add ServiceRestart and TryAgainLater websocket close codes (#828) -- Fix prompt message for `web.run_app` #832 +- Fix prompt message for `web.run_app` (#832) -- Allow to pass None as a timeout value to disable timeout logic #834 +- Allow to pass None as a timeout value to disable timeout logic (#834) -- Fix leak of connection slot during connection error #835 +- Fix leak of connection slot during connection error (#835) - Gunicorn worker with uvloop support - `aiohttp.worker.GunicornUVLoopWebWorker` #878 + `aiohttp.worker.GunicornUVLoopWebWorker` (#878) -- Don't send body in response to HEAD request #838 +- Don't send body in response to HEAD request (#838) -- Skip the preamble in MultipartReader #881 +- Skip the preamble in MultipartReader (#881) -- Implement BasicAuth decode classmethod. #744 +- Implement BasicAuth decode classmethod. (#744) -- Don't crash logger when transport is None #889 +- Don't crash logger when transport is None (#889) - Use a create_future compatibility wrapper instead of creating - Futures directly #896 + Futures directly (#896) -- Add test utilities to aiohttp #902 +- Add test utilities to aiohttp (#902) -- Improve Request.__repr__ #875 +- Improve Request.__repr__ (#875) -- Skip DNS resolving if provided host is already an ip address #874 +- Skip DNS resolving if provided host is already an ip address (#874) -- Add headers to ClientSession.ws_connect #785 +- Add headers to ClientSession.ws_connect (#785) -- Document that server can send pre-compressed data #906 +- Document that server can send pre-compressed data (#906) -- Don't add Content-Encoding and Transfer-Encoding if no body #891 +- Don't add Content-Encoding and Transfer-Encoding if no body (#891) -- Add json() convenience methods to websocket message objects #897 +- Add json() convenience methods to websocket message objects (#897) -- Add client_resp.raise_for_status() #908 +- Add client_resp.raise_for_status() (#908) -- Implement cookie filter #799 +- Implement cookie filter (#799) -- Include an example of middleware to handle error pages #909 +- Include an example of middleware to handle error pages (#909) -- Fix error handling in StaticFileMixin #856 +- Fix error handling in StaticFileMixin (#856) -- Add mocked request helper #900 +- Add mocked request helper (#900) -- Fix empty ALLOW Response header for cls based View #929 +- Fix empty ALLOW Response header for cls based View (#929) -- Respect CONNECT method to implement a proxy server #847 +- Respect CONNECT method to implement a proxy server (#847) -- Add pytest_plugin #914 +- Add pytest_plugin (#914) - Add tutorial - Add backlog option to support more than 128 (default value in - "create_server" function) concurrent connections #892 + "create_server" function) concurrent connections (#892) -- Allow configuration of header size limits #912 +- Allow configuration of header size limits (#912) -- Separate sending file logic from StaticRoute dispatcher #901 +- Separate sending file logic from StaticRoute dispatcher (#901) - Drop deprecated share_cookies connector option (BACKWARD INCOMPATIBLE) @@ -645,28 +646,28 @@ - Drop all mentions about api changes in documentation for versions older than 0.16 -- Allow to override default cookie jar #963 +- Allow to override default cookie jar (#963) - Add manylinux wheel builds -- Dup a socket for sendfile usage #964 +- Dup a socket for sendfile usage (#964) 0.21.6 (05-05-2016) =================== -- Drop initial query parameters on redirects #853 +- Drop initial query parameters on redirects (#853) 0.21.5 (03-22-2016) =================== -- Fix command line arg parsing #797 +- Fix command line arg parsing (#797) 0.21.4 (03-12-2016) =================== - Fix ResourceAdapter: don't add method to allowed if resource is not - match #826 + match (#826) - Fix Resource: append found method to returned allowed methods @@ -674,12 +675,12 @@ =================== - Fix a regression: support for handling ~/path in static file routes was - broken #782 + broken (#782) 0.21.1 (02-10-2016) =================== -- Make new resources classes public #767 +- Make new resources classes public (#767) - Add `router.resources()` view @@ -688,22 +689,22 @@ 0.21.0 (02-04-2016) =================== -- Introduce on_shutdown signal #722 +- Introduce on_shutdown signal (#722) -- Implement raw input headers #726 +- Implement raw input headers (#726) -- Implement web.run_app utility function #734 +- Implement web.run_app utility function (#734) - Introduce on_cleanup signal - Deprecate Application.finish() / Application.register_on_finish() in favor of on_cleanup. -- Get rid of bare aiohttp.request(), aiohttp.get() and family in docs #729 +- Get rid of bare aiohttp.request(), aiohttp.get() and family in docs (#729) -- Deprecate bare aiohttp.request(), aiohttp.get() and family #729 +- Deprecate bare aiohttp.request(), aiohttp.get() and family (#729) -- Refactor keep-alive support #737: +- Refactor keep-alive support (#737): - Enable keepalive for HTTP 1.0 by default @@ -722,18 +723,18 @@ - don't send `Connection` header for HTTP 1.0 - Add version parameter to ClientSession constructor, - deprecate it for session.request() and family #736 + deprecate it for session.request() and family (#736) -- Enable access log by default #735 +- Enable access log by default (#735) - Deprecate app.router.register_route() (the method was not documented intentionally BTW). - Deprecate app.router.named_routes() in favor of app.router.named_resources() -- route.add_static accepts pathlib.Path now #743 +- route.add_static accepts pathlib.Path now (#743) -- Add command line support: `$ python -m aiohttp.web package.main` #740 +- Add command line support: `$ python -m aiohttp.web package.main` (#740) - FAQ section was added to docs. Enjoy and fill free to contribute new topics @@ -741,32 +742,32 @@ - Document ClientResponse's host, method, url properties -- Use CORK/NODELAY in client API #748 +- Use CORK/NODELAY in client API (#748) - ClientSession.close and Connector.close are coroutines now - Close client connection on exception in ClientResponse.release() -- Allow to read multipart parts without content-length specified #750 +- Allow to read multipart parts without content-length specified (#750) -- Add support for unix domain sockets to gunicorn worker #470 +- Add support for unix domain sockets to gunicorn worker (#470) -- Add test for default Expect handler #601 +- Add test for default Expect handler (#601) - Add the first demo project -- Rename `loader` keyword argument in `web.Request.json` method. #646 +- Rename `loader` keyword argument in `web.Request.json` method. (#646) -- Add local socket binding for TCPConnector #678 +- Add local socket binding for TCPConnector (#678) 0.20.2 (01-07-2016) =================== -- Enable use of `await` for a class based view #717 +- Enable use of `await` for a class based view (#717) -- Check address family to fill wsgi env properly #718 +- Check address family to fill wsgi env properly (#718) -- Fix memory leak in headers processing (thanks to Marco Paolini) #723 +- Fix memory leak in headers processing (thanks to Marco Paolini) (#723) 0.20.1 (12-30-2015) =================== @@ -774,7 +775,7 @@ - Raise RuntimeError is Timeout context manager was used outside of task context. -- Add number of bytes to stream.read_nowait #700 +- Add number of bytes to stream.read_nowait (#700) - Use X-FORWARDED-PROTO for wsgi.url_scheme when available @@ -785,19 +786,19 @@ - Extend list of web exceptions, add HTTPMisdirectedRequest, HTTPUpgradeRequired, HTTPPreconditionRequired, HTTPTooManyRequests, HTTPRequestHeaderFieldsTooLarge, HTTPVariantAlsoNegotiates, - HTTPNotExtended, HTTPNetworkAuthenticationRequired status codes #644 + HTTPNotExtended, HTTPNetworkAuthenticationRequired status codes (#644) -- Do not remove AUTHORIZATION header by WSGI handler #649 +- Do not remove AUTHORIZATION header by WSGI handler (#649) -- Fix broken support for https proxies with authentication #617 +- Fix broken support for https proxies with authentication (#617) - Get REMOTE_* and SEVER_* http vars from headers when listening on - unix socket #654 + unix socket (#654) -- Add HTTP 308 support #663 +- Add HTTP 308 support (#663) - Add Tf format (time to serve request in seconds, %06f format) to - access log #669 + access log (#669) - Remove one and a half years long deprecated ClientResponse.read_and_close() method @@ -806,77 +807,77 @@ on sending chunked encoded data - Use TCP_CORK and TCP_NODELAY to optimize network latency and - throughput #680 + throughput (#680) -- Websocket XOR performance improved #687 +- Websocket XOR performance improved (#687) -- Avoid sending cookie attributes in Cookie header #613 +- Avoid sending cookie attributes in Cookie header (#613) - Round server timeouts to seconds for grouping pending calls. That - leads to less amount of poller syscalls e.g. epoll.poll(). #702 + leads to less amount of poller syscalls e.g. epoll.poll(). (#702) -- Close connection on websocket handshake error #703 +- Close connection on websocket handshake error (#703) -- Implement class based views #684 +- Implement class based views (#684) -- Add *headers* parameter to ws_connect() #709 +- Add *headers* parameter to ws_connect() (#709) -- Drop unused function `parse_remote_addr()` #708 +- Drop unused function `parse_remote_addr()` (#708) -- Close session on exception #707 +- Close session on exception (#707) -- Store http code and headers in WSServerHandshakeError #706 +- Store http code and headers in WSServerHandshakeError (#706) -- Make some low-level message properties readonly #710 +- Make some low-level message properties readonly (#710) 0.19.0 (11-25-2015) =================== -- Memory leak in ParserBuffer #579 +- Memory leak in ParserBuffer (#579) - Support gunicorn's `max_requests` settings in gunicorn worker -- Fix wsgi environment building #573 +- Fix wsgi environment building (#573) -- Improve access logging #572 +- Improve access logging (#572) -- Drop unused host and port from low-level server #586 +- Drop unused host and port from low-level server (#586) -- Add Python 3.5 `async for` implementation to server websocket #543 +- Add Python 3.5 `async for` implementation to server websocket (#543) - Add Python 3.5 `async for` implementation to client websocket - Add Python 3.5 `async with` implementation to client websocket -- Add charset parameter to web.Response constructor #593 +- Add charset parameter to web.Response constructor (#593) - Forbid passing both Content-Type header and content_type or charset params into web.Response constructor -- Forbid duplicating of web.Application and web.Request #602 +- Forbid duplicating of web.Application and web.Request (#602) -- Add an option to pass Origin header in ws_connect #607 +- Add an option to pass Origin header in ws_connect (#607) -- Add json_response function #592 +- Add json_response function (#592) -- Make concurrent connections respect limits #581 +- Make concurrent connections respect limits (#581) -- Collect history of responses if redirects occur #614 +- Collect history of responses if redirects occur (#614) -- Enable passing pre-compressed data in requests #621 +- Enable passing pre-compressed data in requests (#621) -- Expose named routes via UrlDispatcher.named_routes() #622 +- Expose named routes via UrlDispatcher.named_routes() (#622) -- Allow disabling sendfile by environment variable AIOHTTP_NOSENDFILE #629 +- Allow disabling sendfile by environment variable AIOHTTP_NOSENDFILE (#629) - Use ensure_future if available -- Always quote params for Content-Disposition #641 +- Always quote params for Content-Disposition (#641) -- Support async for in multipart reader #640 +- Support async for in multipart reader (#640) -- Add Timeout context manager #611 +- Add Timeout context manager (#611) 0.18.4 (13-11-2015) =================== @@ -887,12 +888,12 @@ 0.18.3 (25-10-2015) =================== -- Fix formatting for _RequestContextManager helper #590 +- Fix formatting for _RequestContextManager helper (#590) 0.18.2 (22-10-2015) =================== -- Fix regression for OpenSSL < 1.0.0 #583 +- Fix regression for OpenSSL < 1.0.0 (#583) 0.18.1 (20-10-2015) =================== @@ -904,7 +905,7 @@ =================== - Use errors.HttpProcessingError.message as HTTP error reason and - message #459 + message (#459) - Optimize cythonized multidict a bit @@ -912,27 +913,27 @@ - default headers in ClientSession are now case-insensitive -- Make '=' char and 'wss://' schema safe in urls #477 +- Make '=' char and 'wss://' schema safe in urls (#477) -- `ClientResponse.close()` forces connection closing by default from now #479 +- `ClientResponse.close()` forces connection closing by default from now (#479) N.B. Backward incompatible change: was `.close(force=False) Using `force` parameter for the method is deprecated: use `.release()` instead. -- Properly requote URL's path #480 +- Properly requote URL's path (#480) -- add `skip_auto_headers` parameter for client API #486 +- add `skip_auto_headers` parameter for client API (#486) -- Properly parse URL path in aiohttp.web.Request #489 +- Properly parse URL path in aiohttp.web.Request (#489) -- Raise RuntimeError when chunked enabled and HTTP is 1.0 #488 +- Raise RuntimeError when chunked enabled and HTTP is 1.0 (#488) -- Fix a bug with processing io.BytesIO as data parameter for client API #500 +- Fix a bug with processing io.BytesIO as data parameter for client API (#500) -- Skip auto-generation of Content-Type header #507 +- Skip auto-generation of Content-Type header (#507) -- Use sendfile facility for static file handling #503 +- Use sendfile facility for static file handling (#503) - Default `response_factory` in `app.router.add_static` now is `StreamResponse`, not `None`. The functionality is not changed if @@ -941,17 +942,17 @@ - Drop `ClientResponse.message` attribute, it was always implementation detail. - Streams are optimized for speed and mostly memory in case of a big - HTTP message sizes #496 + HTTP message sizes (#496) - Fix a bug for server-side cookies for dropping cookie and setting it again without Max-Age parameter. -- Don't trim redirect URL in client API #499 +- Don't trim redirect URL in client API (#499) -- Extend precision of access log "D" to milliseconds #527 +- Extend precision of access log "D" to milliseconds (#527) - Deprecate `StreamResponse.start()` method in favor of - `StreamResponse.prepare()` coroutine #525 + `StreamResponse.prepare()` coroutine (#525) `.start()` is still supported but responses begun with `.start()` does not call signal for response preparing to be sent. @@ -959,48 +960,48 @@ - Add `StreamReader.__repr__` - Drop Python 3.3 support, from now minimal required version is Python - 3.4.1 #541 + 3.4.1 (#541) -- Add `async with` support for `ClientSession.request()` and family #536 +- Add `async with` support for `ClientSession.request()` and family (#536) -- Ignore message body on 204 and 304 responses #505 +- Ignore message body on 204 and 304 responses (#505) -- `TCPConnector` processed both IPv4 and IPv6 by default #559 +- `TCPConnector` processed both IPv4 and IPv6 by default (#559) -- Add `.routes()` view for urldispatcher #519 +- Add `.routes()` view for urldispatcher (#519) -- Route name should be a valid identifier name from now #567 +- Route name should be a valid identifier name from now (#567) -- Implement server signals #562 +- Implement server signals (#562) - Drop a year-old deprecated *files* parameter from client API. -- Added `async for` support for aiohttp stream #542 +- Added `async for` support for aiohttp stream (#542) 0.17.4 (09-29-2015) =================== -- Properly parse URL path in aiohttp.web.Request #489 +- Properly parse URL path in aiohttp.web.Request (#489) - Add missing coroutine decorator, the client api is await-compatible now 0.17.3 (08-28-2015) =================== -- Remove Content-Length header on compressed responses #450 +- Remove Content-Length header on compressed responses (#450) - Support Python 3.5 -- Improve performance of transport in-use list #472 +- Improve performance of transport in-use list (#472) -- Fix connection pooling #473 +- Fix connection pooling (#473) 0.17.2 (08-11-2015) =================== -- Don't forget to pass `data` argument forward #462 +- Don't forget to pass `data` argument forward (#462) -- Fix multipart read bytes count #463 +- Fix multipart read bytes count (#463) 0.17.1 (08-10-2015) =================== @@ -1010,28 +1011,28 @@ 0.17.0 (08-04-2015) =================== -- Make StaticRoute support Last-Modified and If-Modified-Since headers #386 +- Make StaticRoute support Last-Modified and If-Modified-Since headers (#386) - Add Request.if_modified_since and Stream.Response.last_modified properties -- Fix deflate compression when writing a chunked response #395 +- Fix deflate compression when writing a chunked response (#395) - Request`s content-length header is cleared now after redirect from - POST method #391 + POST method (#391) -- Return a 400 if server received a non HTTP content #405 +- Return a 400 if server received a non HTTP content (#405) -- Fix keep-alive support for aiohttp clients #406 +- Fix keep-alive support for aiohttp clients (#406) -- Allow gzip compression in high-level server response interface #403 +- Allow gzip compression in high-level server response interface (#403) -- Rename TCPConnector.resolve and family to dns_cache #415 +- Rename TCPConnector.resolve and family to dns_cache (#415) -- Make UrlDispatcher ignore quoted characters during url matching #414 +- Make UrlDispatcher ignore quoted characters during url matching (#414) Backward-compatibility warning: this may change the url matched by - your queries if they send quoted character (like %2F for /) #414 + your queries if they send quoted character (like %2F for /) (#414) -- Use optional cchardet accelerator if present #418 +- Use optional cchardet accelerator if present (#418) - Borrow loop from Connector in ClientSession if loop is not set @@ -1040,50 +1041,50 @@ - Add toplevel get(), post(), put(), head(), delete(), options(), patch() coroutines. -- Fix IPv6 support for client API #425 +- Fix IPv6 support for client API (#425) -- Pass SSL context through proxy connector #421 +- Pass SSL context through proxy connector (#421) - Make the rule: path for add_route should start with slash - Don't process request finishing by low-level server on closed event loop -- Don't override data if multiple files are uploaded with same key #433 +- Don't override data if multiple files are uploaded with same key (#433) - Ensure multipart.BodyPartReader.read_chunk read all the necessary data to avoid false assertions about malformed multipart payload -- Don't send body for 204, 205 and 304 http exceptions #442 +- Don't send body for 204, 205 and 304 http exceptions (#442) -- Correctly skip Cython compilation in MSVC not found #453 +- Correctly skip Cython compilation in MSVC not found (#453) -- Add response factory to StaticRoute #456 +- Add response factory to StaticRoute (#456) -- Don't append trailing CRLF for multipart.BodyPartReader #454 +- Don't append trailing CRLF for multipart.BodyPartReader (#454) 0.16.6 (07-15-2015) =================== -- Skip compilation on Windows if vcvarsall.bat cannot be found #438 +- Skip compilation on Windows if vcvarsall.bat cannot be found (#438) 0.16.5 (06-13-2015) =================== -- Get rid of all comprehensions and yielding in _multidict #410 +- Get rid of all comprehensions and yielding in _multidict (#410) 0.16.4 (06-13-2015) =================== - Don't clear current exception in multidict's `__repr__` (cythonized - versions) #410 + versions) (#410) 0.16.3 (05-30-2015) =================== -- Fix StaticRoute vulnerability to directory traversal attacks #380 +- Fix StaticRoute vulnerability to directory traversal attacks (#380) 0.16.2 (05-27-2015) @@ -1093,26 +1094,26 @@ 3.4.1 instead of 3.4.0 - Add check for presence of loop.is_closed() method before call the - former #378 + former (#378) 0.16.1 (05-27-2015) =================== -- Fix regression in static file handling #377 +- Fix regression in static file handling (#377) 0.16.0 (05-26-2015) =================== -- Unset waiter future after cancellation #363 +- Unset waiter future after cancellation (#363) -- Update request url with query parameters #372 +- Update request url with query parameters (#372) - Support new `fingerprint` param of TCPConnector to enable verifying - SSL certificates via MD5, SHA1, or SHA256 digest #366 + SSL certificates via MD5, SHA1, or SHA256 digest (#366) - Setup uploaded filename if field value is binary and transfer - encoding is not specified #349 + encoding is not specified (#349) - Implement `ClientSession.close()` method @@ -1127,20 +1128,20 @@ - Add `__del__` to client-side objects: sessions, connectors, connections, requests, responses. -- Refactor connections cleanup by connector #357 +- Refactor connections cleanup by connector (#357) -- Add `limit` parameter to connector constructor #358 +- Add `limit` parameter to connector constructor (#358) -- Add `request.has_body` property #364 +- Add `request.has_body` property (#364) -- Add `response_class` parameter to `ws_connect()` #367 +- Add `response_class` parameter to `ws_connect()` (#367) - `ProxyConnector` does not support keep-alive requests by default - starting from now #368 + starting from now (#368) - Add `connector.force_close` property -- Add ws_connect to ClientSession #374 +- Add ws_connect to ClientSession (#374) - Support optional `chunk_size` parameter in `router.add_static()` @@ -1150,7 +1151,7 @@ - Fix graceful shutdown handling -- Fix `Expect` header handling for not found and not allowed routes #340 +- Fix `Expect` header handling for not found and not allowed routes (#340) 0.15.2 (04-19-2015) @@ -1162,15 +1163,15 @@ - Allow to match any request method with `*` -- Explicitly call drain on transport #316 +- Explicitly call drain on transport (#316) -- Make chardet module dependency mandatory #318 +- Make chardet module dependency mandatory (#318) -- Support keep-alive for HTTP 1.0 #325 +- Support keep-alive for HTTP 1.0 (#325) -- Do not chunk single file during upload #327 +- Do not chunk single file during upload (#327) -- Add ClientSession object for cookie storage and default headers #328 +- Add ClientSession object for cookie storage and default headers (#328) - Add `keep_alive_on` argument for HTTP server handler. @@ -1196,13 +1197,13 @@ - Client WebSockets support -- New Multipart system #273 +- New Multipart system (#273) -- Support for "Except" header #287 #267 +- Support for "Except" header (#287) (#267) -- Set default Content-Type for post requests #184 +- Set default Content-Type for post requests (#184) -- Fix issue with construction dynamic route with regexps and trailing slash #266 +- Fix issue with construction dynamic route with regexps and trailing slash (#266) - Add repr to web.Request @@ -1212,7 +1213,7 @@ - Add repr for web.Application -- Add repr to UrlMappingMatchInfo #217 +- Add repr to UrlMappingMatchInfo (#217) - Gunicorn 19.2.x compatibility @@ -1220,29 +1221,29 @@ 0.14.4 (01-29-2015) =================== -- Fix issue with error during constructing of url with regex parts #264 +- Fix issue with error during constructing of url with regex parts (#264) 0.14.3 (01-28-2015) =================== -- Use path='/' by default for cookies #261 +- Use path='/' by default for cookies (#261) 0.14.2 (01-23-2015) =================== -- Connections leak in BaseConnector #253 +- Connections leak in BaseConnector (#253) -- Do not swallow websocket reader exceptions #255 +- Do not swallow websocket reader exceptions (#255) -- web.Request's read, text, json are memorized #250 +- web.Request's read, text, json are memorized (#250) 0.14.1 (01-15-2015) =================== -- HttpMessage._add_default_headers does not overwrite existing headers #216 +- HttpMessage._add_default_headers does not overwrite existing headers (#216) - Expose multidict classes at package level @@ -1281,21 +1282,21 @@ - Server has 75 seconds keepalive timeout now, was non-keepalive by default. -- Application does not accept `**kwargs` anymore (#243). +- Application does not accept `**kwargs` anymore ((#243)). - Request is inherited from dict now for making per-request storage to - middlewares (#242). + middlewares ((#242)). 0.13.1 (12-31-2014) =================== -- Add `aiohttp.web.StreamResponse.started` property #213 +- Add `aiohttp.web.StreamResponse.started` property (#213) - HTML escape traceback text in `ServerHttpProtocol.handle_error` - Mention handler and middlewares in `aiohttp.web.RequestHandler.handle_request` - on error (#218) + on error ((#218)) 0.13.0 (12-29-2014) @@ -1305,16 +1306,16 @@ - Chain exceptions when raise `ClientRequestError`. -- Support custom regexps in route variables #204 +- Support custom regexps in route variables (#204) - Fixed graceful shutdown, disable keep-alive on connection closing. - Decode HTTP message with `utf-8` encoding, some servers send headers - in utf-8 encoding #207 + in utf-8 encoding (#207) -- Support `aiohtt.web` middlewares #209 +- Support `aiohtt.web` middlewares (#209) -- Add ssl_context to TCPConnector #206 +- Add ssl_context to TCPConnector (#206) 0.12.0 (12-12-2014) @@ -1324,7 +1325,7 @@ Sorry, we have to do this. - Automatically force aiohttp.web handlers to coroutines in - `UrlDispatcher.add_route()` #186 + `UrlDispatcher.add_route()` (#186) - Rename `Request.POST()` function to `Request.post()` @@ -1353,15 +1354,15 @@ 0.11.0 (11-29-2014) =================== -- Support named routes in `aiohttp.web.UrlDispatcher` #179 +- Support named routes in `aiohttp.web.UrlDispatcher` (#179) -- Make websocket subprotocols conform to spec #181 +- Make websocket subprotocols conform to spec (#181) 0.10.2 (11-19-2014) =================== -- Don't unquote `environ['PATH_INFO']` in wsgi.py #177 +- Don't unquote `environ['PATH_INFO']` in wsgi.py (#177) 0.10.1 (11-17-2014) @@ -1387,54 +1388,54 @@ from 'Can not read status line' to explicit 'Connection closed by server' -- Drop closed connections from connector #173 +- Drop closed connections from connector (#173) -- Set server.transport to None on .closing() #172 +- Set server.transport to None on .closing() (#172) 0.9.3 (10-30-2014) ================== -- Fix compatibility with asyncio 3.4.1+ #170 +- Fix compatibility with asyncio 3.4.1+ (#170) 0.9.2 (10-16-2014) ================== -- Improve redirect handling #157 +- Improve redirect handling (#157) -- Send raw files as is #153 +- Send raw files as is (#153) -- Better websocket support #150 +- Better websocket support (#150) 0.9.1 (08-30-2014) ================== -- Added MultiDict support for client request params and data #114. +- Added MultiDict support for client request params and data (#114). -- Fixed parameter type for IncompleteRead exception #118. +- Fixed parameter type for IncompleteRead exception (#118). -- Strictly require ASCII headers names and values #137 +- Strictly require ASCII headers names and values (#137) -- Keep port in ProxyConnector #128. +- Keep port in ProxyConnector (#128). -- Python 3.4.1 compatibility #131. +- Python 3.4.1 compatibility (#131). 0.9.0 (07-08-2014) ================== -- Better client basic authentication support #112. +- Better client basic authentication support (#112). -- Fixed incorrect line splitting in HttpRequestParser #97. +- Fixed incorrect line splitting in HttpRequestParser (#97). - Support StreamReader and DataQueue as request data. -- Client files handling refactoring #20. +- Client files handling refactoring (#20). - Backward incompatible: Replace DataQueue with StreamReader for - request payload #87. + request payload (#87). 0.8.4 (07-04-2014) @@ -1557,7 +1558,7 @@ - Better support for server exit. -- Read response body until EOF if content-length is not defined #14 +- Read response body until EOF if content-length is not defined (#14) 0.6.2 (02-18-2014) From af72aa18b4ee329e70a97ec75fe4ce5e10edc62d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 15:43:39 +0300 Subject: [PATCH 033/167] Improve ReST markup --- docs/web_reference.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/web_reference.rst b/docs/web_reference.rst index ef94e957961..8e2b5696bbe 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -2198,13 +2198,18 @@ Normalize path middleware - Double slashes are replaced by one. The middleware returns as soon as it finds a path that resolves - correctly. The order if all enabled is 1) merge_slashes, 2) append_slash - and 3) both merge_slashes and append_slash. If the path resolves with - at least one of those conditions, it will redirect to the new path. + correctly. The order if all enabled is: - If *append_slash* is True append slash when needed. If a resource is + 1. *merge_slashes* + 2. *append_slash* + 3. both *merge_slashes* and *append_slash* + + If the path resolves with at least one of those conditions, it will + redirect to the new path. + + If *append_slash* is ``True`` append slash when needed. If a resource is defined with trailing slash and the request comes without it, it will append it automatically. - If *merge_slashes* is True, merge multiple consecutive slashes in the + If *merge_slashes* is ``True``, merge multiple consecutive slashes in the path into one. From 918fe4c520812b4b8b5e6f927dd6554ad3917031 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 26 Jun 2017 17:18:04 +0300 Subject: [PATCH 034/167] Polish docs --- aiohttp/client.py | 2 +- aiohttp/client_exceptions.py | 12 +-- docs/client_reference.rst | 150 +++++++++++++++++++++++++++++------ 3 files changed, 133 insertions(+), 31 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index de1310baeb5..9698ddc5d9c 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -277,7 +277,7 @@ def _request(self, method, url, *, if r_url is None: raise ClientRedirectError( resp.request_info, - history, + resp.history, resp.status) r_url = URL( r_url, encoded=not self.requote_redirect_url) diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 0e9e682ac96..e26d2a73ad4 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -1,6 +1,6 @@ """HTTP related errors.""" -from asyncio import TimeoutError +import asyncio __all__ = ( @@ -49,10 +49,6 @@ def __init__(self, request_info, history, code): message="Response has no Location or URI header") -class ClientPayloadError(ClientError): - """Response payload error.""" - - class WSServerHandshakeError(ClientResponseError): """websocket server handshake error.""" @@ -101,7 +97,7 @@ def __init__(self, message=None): self.message = message -class ServerTimeoutError(ServerConnectionError, TimeoutError): +class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError): """Server timeout error.""" @@ -118,3 +114,7 @@ def __repr__(self): return '<{} expected={} got={} host={} port={}>'.format( self.__class__.__name__, self.expected, self.got, self.host, self.port) + + +class ClientPayloadError(ClientError): + """Response payload error.""" diff --git a/docs/client_reference.rst b/docs/client_reference.rst index e1464a077d1..48423359842 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1403,7 +1403,7 @@ CookieJar Client exceptions -^^^^^^^^^^^^^^^^^ +----------------- Exception hierarchy has been significantly modified in version 2.0. aiohttp defines only exceptions that covers connection handling @@ -1418,43 +1418,145 @@ chunks or not enough data that satisfy the content-length header. All exceptions are available as members of *aiohttp* module. -Hierarchy of exceptions: +.. exception:: ClientError + + Base class for all client specific exceptions. + + Derived from :exc:`Exception` + + +Response errors +^^^^^^^^^^^^^^^ + +.. exception:: ClientResponseError + + These exceptions could happen after we get response from server. + + Derived from :exc:`ClientError` + + .. attribute:: request_info + + Instance of :class:`RequestInfo` object, contains information + about request. + + .. attribute:: history + + History from failed response, if available, else empty tuple. + + A :class:`tuple` of :class:`ClientResponse` objects used for + handle redirection responses. + + +.. class:: ClientRedirectError + + Response is a redirect but ``Location`` or ``URI`` headers are missing. + + Derived from :exc:`ClientResponseError` + +.. class:: WSServerHandshakeError + + Web socket server response error. + + Derived from :exc:`ClientResponseError` + + +.. class:: ClientHttpProxyError + + Proxy response error. + + Derived from :exc:`ClientResponseError` + +Connection errors +^^^^^^^^^^^^^^^^^ + +.. class:: ClientConnectionError + + These exceptions related to low-level connection problems. + + + Derived from :exc:`ClientError` + +.. class:: ClientOSError + + Subset of connection errors that are initiated by an :exc:`OSError` + exception. + + Derived from :exc:`ClientConnectionError` and :exc:`OSError` + +.. class:: ClientConnectorError + + Connector related exceptions. + + Derived from :exc:`ClientOSError` + +.. class:: ClientProxyConnectionError + + Derived from :exc:`ClientConnectonError` + +.. class:: ServerConnectionError + + Derived from :exc:`ClientConnectonError` + + +.. class:: ServerDisconnectedError + + Server disconnected. + + Derived from :exc:`ServerDisconnectonError` + + .. attribute:: message + + Partially parsed HTTP message (optional). + + +.. class:: ServerTimeoutError + + Server operation timeout: read timeout, etc. + + Derived from :exc:`ServerConnectonError` and :exc:`asyncio.TimeoutError` + +.. class:: ServerFingerprintMismatch + + Server fingerprint mismatch. + + Derived from :exc:`ServerConnectonError` -* :exc:`aiohttp.ClientError` - Base class for all client specific exceptions - - :exc:`aiohttp.ClientResponseError` - exceptions that could happen after - we get response from server. +.. class:: ClientPayloadError - `request_info` - Instance of `RequestInfo` object, contains - information about request. + This exception can only be raised while reading the response + payload if one of these errors occurs: - `history` - History from `ClientResponse` object, if available, else empty tuple. + 1. invalid compression + 2. malformed chunked encoding + 3. not enough data that satisfy ``Content-Length`` HTTP header. - - `aiohttp.ClientRedirectError` - Response is a redirect but - ``Location`` or ``URI`` headers are missing. + Derived from :exc:`ClientError` - - `aiohttp.WSServerHandshakeError` - web socket server response error +Hierarchy of exceptions +^^^^^^^^^^^^^^^^^^^^^^^ - - `aiohttp.ClientHttpProxyError` - proxy response +* :exc:`ClientError` - - `aiohttp.ClientConnectionError` - exceptions related to low-level connection problems + * :exc:`ClientResponseError` - - `aiohttp.ClientOSError` - subset of connection errors that are initiated by an OSError exception + * :exc:`ClientRedirectError` + * :exc:`WSServerHandshakeError` + * :exc:`ClientHttpProxyError` - - `aiohttp.ClientConnectorError` - connector related exceptions + * :exc:`ClientConnectionError` - - `aiohttp.ClientProxyConnectionError` - proxy connection initialization error + * :exc:`ClientOSError` - - `aiohttp.ServerConnectionError` - server connection related errors + * :exc:`ClientConnectorError` - - `aiohttp.ServerDisconnectedError` - server disconnected + * :exc:`ClientProxyConnectionError` - `message` - Partially parsed http message (optional) + * :exc:`ServerConnectionError` - - `aiohttp.ServerTimeoutError` - server operation timeout, (read timeout, etc) + * :exc:`ServerDisconnectedError` + * :exc:`ServerTimeoutError` - - `aiohttp.ServerFingerprintMismatch` - server fingerprint mismatch + * :exc:`ServerFingerprintMismatch` - - `aiohttp.ClientPayloadError` - This exception can only be raised while reading the response - payload if one of these errors occurs: invalid compression, malformed chunked encoding or - not enough data that satisfy content-length header. + * :exc:`ClientPayloadError` From 0957977050023ebb375fad29043a6f1f086a185b Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Tue, 27 Jun 2017 00:05:44 +0200 Subject: [PATCH 035/167] Update yarl from 0.10.3 to 0.11.0 (#2020) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index ee547ec05fc..355f6f6c087 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -18,5 +18,5 @@ gunicorn==19.7.1 pygments>=2.1 #aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 -yarl==0.10.3 +yarl==0.11.0 -e . From 6ba5d61e1fa056e4c5816c4aae204127b806446d Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Wed, 28 Jun 2017 13:19:12 +0200 Subject: [PATCH 036/167] Improve tox ini (#2031) * Improve tox.ini to be able to run only some tests This change will make tox use posargs if there are any or fallback to "tests" args (which will run the all the tests). You can now use this command to start a specific test: tox -e py35 -- -x tests/test_client_connection.py * Add Python 3.6 to the testing environment --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 808f4f77db7..65099b9026f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] -envlist = check, {py34,py35}-{debug,release}-{cchardet,cython,pure}, report +envlist = check, {py34,py35,py36}-{debug,release}-{cchardet,cython,pure}, report [testenv] @@ -17,7 +17,7 @@ deps = commands = # --cov={envsitepackagesdir}/tests # py.test --cov={envsitepackagesdir}/aiohttp tests {posargs} - coverage run -m pytest tests {posargs} + coverage run -m pytest {posargs:tests} mv .coverage .coverage.{envname} setenv = @@ -27,6 +27,7 @@ setenv = basepython: py34: python3.4 py35: python3.5 + py36: python3.6 whitelist_externals = coverage @@ -47,7 +48,7 @@ commands = coverage erase basepython: - python3.5 + python3.6 [testenv:report] @@ -58,4 +59,4 @@ commands = echo "open file://{toxinidir}/coverage/index.html" basepython: - python3.5 + python3.6 From afbff7b30fee30cc1e52abeb23a82d08b2396a36 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Wed, 28 Jun 2017 14:31:04 +0200 Subject: [PATCH 037/167] Fix bad variable used (#2034) --- aiohttp/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 606f9743317..8b55c1fdd50 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -401,7 +401,7 @@ def tearDown(self): @asyncio.coroutine def _get_client(self, app): """Return a TestClient instance.""" - return TestClient(self.app, loop=self.loop) + return TestClient(app, loop=self.loop) def unittest_run_loop(func, *args, **kwargs): From 384a183bed0123d8a1d595bb9031301ee08b50af Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 29 Jun 2017 02:12:43 -0700 Subject: [PATCH 038/167] fix for redirect issue (#2030) * fix for http://github.com/aio-libs/aiohttp/issues/2022 - fixes Makefile to run when you have py2 + py3 installed - removes ClientRedirectError - ran isort based on recommendations from make test * apparently local isort is different than remote isort * revert changes * revert change * add new changes file * update based on review --- aiohttp/client.py | 10 ++++------ aiohttp/client_exceptions.py | 14 +------------- changes/2009.feature | 3 --- changes/2030.feature | 2 ++ docs/client_reference.rst | 7 ------- tests/test_client_functional.py | 13 +++++++------ 6 files changed, 14 insertions(+), 35 deletions(-) delete mode 100644 changes/2009.feature create mode 100644 changes/2030.feature diff --git a/aiohttp/client.py b/aiohttp/client.py index 9698ddc5d9c..92f01c2c364 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -15,8 +15,7 @@ from . import connector as connector_mod from . import client_exceptions, client_reqrep, hdrs, http, payload from .client_exceptions import * # noqa -from .client_exceptions import (ClientError, ClientOSError, - ClientRedirectError, ServerTimeoutError, +from .client_exceptions import (ClientError, ClientOSError, ServerTimeoutError, WSServerHandshakeError) from .client_reqrep import * # noqa from .client_reqrep import ClientRequest, ClientResponse @@ -275,10 +274,9 @@ def _request(self, method, url, *, r_url = (resp.headers.get(hdrs.LOCATION) or resp.headers.get(hdrs.URI)) if r_url is None: - raise ClientRedirectError( - resp.request_info, - resp.history, - resp.status) + # see github.com/aio-libs/aiohttp/issues/2022 + break + r_url = URL( r_url, encoded=not self.requote_redirect_url) diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index e26d2a73ad4..057d484904a 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -12,7 +12,7 @@ 'ServerConnectionError', 'ServerTimeoutError', 'ServerDisconnectedError', 'ServerFingerprintMismatch', - 'ClientResponseError', 'ClientRedirectError', 'ClientPayloadError', + 'ClientResponseError', 'ClientPayloadError', 'ClientHttpProxyError', 'WSServerHandshakeError') @@ -37,18 +37,6 @@ def __init__(self, request_info, history, *, super().__init__("%s, message='%s'" % (code, message)) -class ClientRedirectError(ClientResponseError): - """Redirection error. - - Response is a redirect but Location or URI HTTP headers are - missing - - """ - def __init__(self, request_info, history, code): - super().__init__(request_info, history, code=code, - message="Response has no Location or URI header") - - class WSServerHandshakeError(ClientResponseError): """websocket server handshake error.""" diff --git a/changes/2009.feature b/changes/2009.feature deleted file mode 100644 index c9b57d9445c..00000000000 --- a/changes/2009.feature +++ /dev/null @@ -1,3 +0,0 @@ -`ClientRedirectLError` is raised instead of RuntimeError - -Raise `ClientRedirectError` if a redirect response has no Location or URI HTTP header. diff --git a/changes/2030.feature b/changes/2030.feature new file mode 100644 index 00000000000..542da590eba --- /dev/null +++ b/changes/2030.feature @@ -0,0 +1,2 @@ +Responses to redirects without Location header are returned instead of raising +a RuntimeError \ No newline at end of file diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 48423359842..2cee934e9f3 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1447,12 +1447,6 @@ Response errors handle redirection responses. -.. class:: ClientRedirectError - - Response is a redirect but ``Location`` or ``URI`` headers are missing. - - Derived from :exc:`ClientResponseError` - .. class:: WSServerHandshakeError Web socket server response error. @@ -1540,7 +1534,6 @@ Hierarchy of exceptions * :exc:`ClientResponseError` - * :exc:`ClientRedirectError` * :exc:`WSServerHandshakeError` * :exc:`ClientHttpProxyError` diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index fe3f95d04b3..48bcc9a2971 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -12,7 +12,7 @@ from multidict import MultiDict import aiohttp -from aiohttp import ClientRedirectError, ServerFingerprintMismatch, hdrs, web +from aiohttp import ServerFingerprintMismatch, hdrs, web from aiohttp.helpers import create_future from aiohttp.multipart import MultipartWriter @@ -2097,18 +2097,19 @@ def redirect(request): @asyncio.coroutine def test_redirect_without_location_header(loop, test_client): + body = b'redirect' + @asyncio.coroutine def handler_redirect(request): - return web.Response(status=301) + return web.Response(status=301, body=body) app = web.Application() app.router.add_route('GET', '/redirect', handler_redirect) client = yield from test_client(app) - with pytest.raises(ClientRedirectError) as ctx: - yield from client.get('/redirect') - expected_msg = 'Response has no Location or URI header' - assert str(ctx.value.message) == expected_msg + resp = yield from client.get('/redirect') + data = yield from resp.read() + assert data == body @asyncio.coroutine From 47d077e06c597ea140d050846f761238550d7208 Mon Sep 17 00:00:00 2001 From: Andrew Lytvyn Date: Thu, 29 Jun 2017 17:08:51 +0300 Subject: [PATCH 039/167] Awaitable _CoroGuard and fix `ClientSession.close` warning message (#2027) * add __await__ method for _CoroGuard * fix _CoroGuard checking: raise DeprecationWarning if coro was awaited * revert changes for _CoroGuard(checking if coroutine was awaited). Fix warning message * add tests for await ClientSession().close * update ClientSession.close warning message * add 2027.feature record to changes folder * downgrade changes filename from 2027 to 2026 * trigger travis build --- CONTRIBUTORS.txt | 1 + aiohttp/client.py | 2 +- aiohttp/helpers.py | 5 +++++ changes/2026.feature | 1 + tests/test_py35/test_client.py | 7 +++++++ 5 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 changes/2026.feature diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 40e9f3ca9a6..d56ed9d096d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -26,6 +26,7 @@ Amy Boyle Andrei Ursulenko Andrej Antonov Andrew Leech +Andrew Lytvyn Andrew Svetlov Andrii Soldatenko Anton Kasyanov diff --git a/aiohttp/client.py b/aiohttp/client.py index 92f01c2c364..7b252b8ffe8 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -533,7 +533,7 @@ def close(self): self._connector.close() self._connector = None - return deprecated_noop('ClientSession.close() is not coroutine') + return deprecated_noop('ClientSession.close() is a coroutine') @property def closed(self): diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 4f1f7df2e89..f484591f075 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -66,6 +66,11 @@ def __iter__(self): self._awaited = True return self._coro.__iter__() + if PY_35: + def __await__(self): + self._awaited = True + return (yield from self._coro) + def __del__(self): self._coro = None if not self._awaited: diff --git a/changes/2026.feature b/changes/2026.feature new file mode 100644 index 00000000000..4e352e49cd5 --- /dev/null +++ b/changes/2026.feature @@ -0,0 +1 @@ +Make _CoroGuard awaitable and fix `ClientSession.close` warning message diff --git a/tests/test_py35/test_client.py b/tests/test_py35/test_client.py index e33f657eb4a..0a4a852248d 100644 --- a/tests/test_py35/test_client.py +++ b/tests/test_py35/test_client.py @@ -13,6 +13,13 @@ async def test_async_with_session(loop): assert session.closed +async def test_session_close_awaitable(loop): + session = aiohttp.ClientSession(loop=loop) + await session.close() + + assert session.closed + + async def test_close_resp_on_error_async_with_session(loop, test_server): async def handler(request): resp = web.StreamResponse(headers={'content-length': '100'}) From 3c38baa67e4e5b64764bb749f8cf4da33413af95 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 29 Jun 2017 19:26:13 +0300 Subject: [PATCH 040/167] Mention gitter and StackOverflow in communication channels --- CONTRIBUTING.rst | 8 ++++++-- README.rst | 12 ++++++++++-- docs/index.rst | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ee612235ceb..230b8927ba6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,3 +1,5 @@ +.. _aiohttp-contributing: + Contributing ============ @@ -18,9 +20,11 @@ Workflow is pretty straightforward: 3. Make sure all tests passed - 4. Commit changes to own aiohttp clone + 4. Add a file into ``changes`` folder (:ref:`aiohttp-contributing-changes`). + + 5. Commit changes to own aiohttp clone - 5. Make pull request from github page for your clone against master branch + 6. Make pull request from github page for your clone against master branch .. note:: If your PR has long history or many commits diff --git a/README.rst b/README.rst index f9e3a507653..dc7de6cbae1 100644 --- a/README.rst +++ b/README.rst @@ -136,11 +136,19 @@ Documentation https://aiohttp.readthedocs.io/ -Discussion list -=============== +Communication channels +====================== *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs +Feel free to post your questions and ideas here. + +*gitter chat* https://gitter.im/aio-libs/Lobby + +We support `Stack Overflow +`_. +Please add *aiohttp* tag to your question there. + Requirements ============ diff --git a/docs/index.rst b/docs/index.rst index 97f52a2dea9..0b1d424a19e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -143,13 +143,19 @@ Dependencies $ pip install aiodns -Discussion list ---------------- +Communication channels +---------------------- *aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs Feel free to post your questions and ideas here. +*gitter chat* https://gitter.im/aio-libs/Lobby + +We support `Stack Overflow +`_. +Please add *aiohttp* tag to your question there. + Contributing ------------ From 639ebedd18afa39cd43f6a0734d7d9a98441bdf5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 29 Jun 2017 19:27:43 +0300 Subject: [PATCH 041/167] Convert session.close() back to coroutine in documentation() --- docs/client_reference.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 2cee934e9f3..71c8d52b1e9 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -491,14 +491,12 @@ The client session supports the context manager protocol for self closing. URLs may be either :class:`str` or :class:`~yarl.URL` - .. method:: close() + .. comethod:: close() Close underlying connector. Release all acquired resources. - .. versionchanged:: 2.0 - .. method:: detach() Detach connector from session without closing the former. From 91219239e30f37bed44817af8b77479205199aa6 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 29 Jun 2017 19:54:42 +0300 Subject: [PATCH 042/167] Fix sphinx markup --- CONTRIBUTING.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 230b8927ba6..f249306e142 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,5 +1,3 @@ -.. _aiohttp-contributing: - Contributing ============ From 65c4328ffb8ddc3b9eb95d0f51c757cb9247ad79 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 29 Jun 2017 20:32:16 +0300 Subject: [PATCH 043/167] Fix docs --- CONTRIBUTING.rst | 35 ++++++++++++++++++++++++++++++++++- docs/spelling_wordlist.txt | 25 ++++++++++++++----------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f249306e142..ea800bc7502 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -18,7 +18,7 @@ Workflow is pretty straightforward: 3. Make sure all tests passed - 4. Add a file into ``changes`` folder (:ref:`aiohttp-contributing-changes`). + 4. Add a file into ``changes`` folder (`Changelog update`_). 5. Commit changes to own aiohttp clone @@ -153,6 +153,39 @@ To run spell checker on Linux box you should install it first: $ sudo apt-get install enchant $ pip install sphinxcontrib-spelling +Changelog update +---------------- + +The ``CHANGES.rst`` file is managed using `towncrier +`_ tool and all non trivial +changes must be accompanied by a news entry. + +To add an entry to the news file, first you need to have created an +issue describing the change you want to make. A Pull Request itself +*may* function as such, but it is preferred to have a dedicated issue +(for example, in case the PR ends up rejected due to code quality +reasons). + +Once you have an issue or pull request, you take the number and you +create a file inside of the ``changes/`` directory named after that +issue number with an extension of ``.removal``, ``.feature``, +``.bugfix``, or ``.doc``. Thus if your issue or PR number is ``1234`` and +this change is fixing a bug, then you would create a file +``changes/1234.bugfix``. PRs can span multiple categories by creating +multiple files (for instance, if you added a feature and +deprecated/removed the old feature at the same time, you would create +``changes/NNNN.feature`` and ``changes/NNNN.removal``). Likewise if a PR touches +multiple issues/PRs you may create a file for each of them with the +exact same contents and *Towncrier* will deduplicate them. + +The contents of this file are *reStructuredText* formatted text that +will be used as the content of the news file entry. You do not need to +reference the issue or PR numbers here as *towncrier* will automatically +add a reference to all of the affected issues when rendering the news +file. + + + The End ------- diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index f8c35bcff5d..298bf9f6467 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -16,17 +16,18 @@ autodetection autogenerates autogeneration awaitable -BaseEventLoop backend backends +BaseEventLoop basename BasicAuth BodyPartReader +botocore builtin BytesIO -botocore cchardet cChardet +Changelog charset charsetdetect chunked @@ -43,13 +44,14 @@ CookieJar coroutine Coroutine coroutines +cpu css ctor Ctrl cythonized -cpu -DER de +deduplicate +DER dict Dict django @@ -82,8 +84,8 @@ HTTPException HttpProcessingError httpretty https -incapsulates impl +incapsulates Indices infos inline @@ -95,8 +97,8 @@ IPv ish iterable iterables -Jinja javascript +Jinja json keepalive keepalived @@ -144,12 +146,13 @@ pathlib ping pluggable plugin -Postgres poller pong +Postgres pre programmatically proxied +PRs pubsub py pyenv @@ -159,8 +162,8 @@ readonly readpayload rebase Redis -Refactor refactor +Refactor refactored refactoring regex @@ -174,6 +177,7 @@ RequestContextManager requote resolvers reusage +Runit sa schemas sendfile @@ -189,15 +193,14 @@ subpackage subprotocol subprotocols subtype -Supervisord supervisord -Systemd -Runit +Supervisord Svetlov symlink symlinks syscall syscalls +Systemd TCP teardown Teardown From 587e191830a8634156d638df786f250b7e721e03 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 30 Jun 2017 13:18:07 +0300 Subject: [PATCH 044/167] Extract doc requirements --- requirements/ci.txt | 4 +--- requirements/doc.txt | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 requirements/doc.txt diff --git a/requirements/ci.txt b/requirements/ci.txt index 355f6f6c087..71694e6f444 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,21 +1,19 @@ +-r doc.txt pip==9.0.1 flake8==3.3.0 pyflakes==1.5.0 coverage==4.4.1 cchardet==2.1.0 -sphinx==1.6.2 cython==0.25.2 chardet==3.0.4 isort==4.2.15 tox==2.7.0 multidict==3.1.0 async-timeout==1.2.1 -sphinxcontrib-asyncio==0.2.0 pytest==3.1.2 pytest-cov==2.5.1 pytest-mock==1.6.0 gunicorn==19.7.1 -pygments>=2.1 #aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 yarl==0.11.0 diff --git a/requirements/doc.txt b/requirements/doc.txt new file mode 100644 index 00000000000..9f10577695e --- /dev/null +++ b/requirements/doc.txt @@ -0,0 +1,3 @@ +sphinx==1.6.2 +sphinxcontrib-asyncio==0.2.0 +pygments>=2.1 From 3f711b3dc56d77ab7cc5b671006fad6044b1ab4c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 30 Jun 2017 13:46:36 +0300 Subject: [PATCH 045/167] Add powered by aiohttp doc page --- docs/built_with.rst | 5 +++-- docs/external.rst | 11 +++++++++++ docs/index.rst | 3 +-- docs/powered_by.rst | 12 ++++++++++++ docs/third_party.rst | 4 ++-- 5 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 docs/external.rst create mode 100644 docs/powered_by.rst diff --git a/docs/built_with.rst b/docs/built_with.rst index 1a458f4c6a3..c1bb49dce18 100644 --- a/docs/built_with.rst +++ b/docs/built_with.rst @@ -1,3 +1,5 @@ +.. _aiohttp-built-with: + Built with aiohttp ================== @@ -16,5 +18,4 @@ You can also add a **Built with aiohttp** link somewhere in your project, pointing to ``_. -- `Molotov `_ Load testing tool. - +* `Molotov `_ Load testing tool. diff --git a/docs/external.rst b/docs/external.rst new file mode 100644 index 00000000000..36c08641520 --- /dev/null +++ b/docs/external.rst @@ -0,0 +1,11 @@ +External resources +================== + +Collection of useful external links + + +.. toctree:: + + third_party + built_with + powered_by diff --git a/docs/index.rst b/docs/index.rst index 0b1d424a19e..63d30c2022c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -214,8 +214,7 @@ Contents testing deployment faq - third_party - built_with + external essays contributing changes diff --git a/docs/powered_by.rst b/docs/powered_by.rst new file mode 100644 index 00000000000..b487e9c4bd1 --- /dev/null +++ b/docs/powered_by.rst @@ -0,0 +1,12 @@ +.. _aiohttp-powered-by: + +Powered by aiohttp +================== + +Web sites powered by aiohttp. + +Feel free to fork documentation on github, add a link to your site and +make a Pull Request! + +* https://www.farmersbusinessnetwork.com +* KeepSafe https://www.getkeepsafe.com/ diff --git a/docs/third_party.rst b/docs/third_party.rst index 8e47d00f321..d1e4cabc4b6 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -14,9 +14,9 @@ This page is a list of these tools. Please feel free to add your open sourced library if it's not enlisted yet by making Pull Request to https://github.com/aio-libs/aiohttp/ -- Q. Why do you might want to include your awesome library into the list? +* Why do you might want to include your awesome library into the list? -- A. Just because the list increases your library visibility. People +* Just because the list increases your library visibility. People will have an easy way to find it. From b73ceac303a812fad26914a5b74c5cab660f7164 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 30 Jun 2017 13:59:51 +0300 Subject: [PATCH 046/167] Metnion external resources in README --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index dc7de6cbae1..793e80028cd 100644 --- a/README.rst +++ b/README.rst @@ -136,6 +136,19 @@ Documentation https://aiohttp.readthedocs.io/ +External links +============== + +* `Third party libraries + `_ +* `Built with aiohttp + http://aiohttp.readthedocs.io/en/latest/built_with.html`_ +* `Powered by aiohttp + http://aiohttp.readthedocs.io/en/latest/powered_by.html`_ + +Feel free to make a Pull Request for adding your link to these pages! + + Communication channels ====================== From 26df4ddef51a524db7b8f4b5c343bdd12ea8611e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 30 Jun 2017 14:00:31 +0300 Subject: [PATCH 047/167] Fix markup --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 793e80028cd..31662592df3 100644 --- a/README.rst +++ b/README.rst @@ -142,9 +142,9 @@ External links * `Third party libraries `_ * `Built with aiohttp - http://aiohttp.readthedocs.io/en/latest/built_with.html`_ + `_ * `Powered by aiohttp - http://aiohttp.readthedocs.io/en/latest/powered_by.html`_ + `_ Feel free to make a Pull Request for adding your link to these pages! From e1f8183762b876e39b9f36b0e5f925e98b1cab89 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 30 Jun 2017 21:02:49 +0300 Subject: [PATCH 048/167] Use python_requires in setup.py --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 9dc9ebf4de4..464f1db3c17 100644 --- a/setup.py +++ b/setup.py @@ -62,9 +62,6 @@ def build_extension(self, ext): install_requires = ['chardet', 'multidict>=2.1.4', 'async_timeout>=1.2.0', 'yarl>=0.10.0,<0.11'] -if sys.version_info < (3, 4, 2): - raise RuntimeError("aiohttp requires Python 3.4.2+") - def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() @@ -75,7 +72,6 @@ class PyTest(TestCommand): def run(self): import subprocess - import sys errno = subprocess.call([sys.executable, '-m', 'pytest', 'tests']) raise SystemExit(errno) @@ -111,6 +107,7 @@ def run(self): url='https://github.com/aio-libs/aiohttp/', license='Apache 2', packages=['aiohttp'], + python_requires='>=3.4.2', install_requires=install_requires, tests_require=tests_require, include_package_data=True, From 2a893a81d048086350e7024d6665bdb9963badea Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 30 Jun 2017 14:55:32 -0700 Subject: [PATCH 049/167] Add Home Assistant to powered by page (#2042) --- docs/powered_by.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index b487e9c4bd1..f2e52ac7d2d 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -9,4 +9,5 @@ Feel free to fork documentation on github, add a link to your site and make a Pull Request! * https://www.farmersbusinessnetwork.com +* `Home Assistant `_ * KeepSafe https://www.getkeepsafe.com/ From 8b0272dda6250986ba591b7b6dbaffb22bfbd99d Mon Sep 17 00:00:00 2001 From: Dmitry Chaplinsky Date: Sat, 1 Jul 2017 00:56:37 +0300 Subject: [PATCH 050/167] Update third_party.rst (#2043) --- docs/third_party.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/third_party.rst b/docs/third_party.rst index d1e4cabc4b6..3483710769c 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -118,6 +118,12 @@ period ask to raise he status. - `aiohttp-swagger `_ Swagger API Documentation builder for aiohttp server. +- `aiohttp-swaggerify `_ + Library to automatically generate swagger2.0 definition for aiohttp endpoints. + +- `aiohttp-validate `_ + Simple library that helps you validate your API endpoints requests/responses with json schema. + - `raven-aiohttp `_ An aiohttp transport for raven-python (Sentry client). From c450970c117ad2217c3a74a24c1496c1077e8e38 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 1 Jul 2017 10:17:41 +0300 Subject: [PATCH 051/167] Upgrade deps: multidict 3.0+, yarl 0.11+ (#2040) --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 464f1db3c17..35856f68551 100644 --- a/setup.py +++ b/setup.py @@ -59,8 +59,8 @@ def build_extension(self, ext): raise RuntimeError('Unable to determine version.') -install_requires = ['chardet', 'multidict>=2.1.4', - 'async_timeout>=1.2.0', 'yarl>=0.10.0,<0.11'] +install_requires = ['chardet', 'multidict>=3.0.0', + 'async_timeout>=1.2.0', 'yarl>=0.11.0'] def read(f): From 59b31df419a83a2454738d0c0f914c1dd2cd41a4 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sat, 1 Jul 2017 15:28:14 +0200 Subject: [PATCH 052/167] Update cchardet from 2.1.0 to 2.1.1 (#2046) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 71694e6f444..327ae80ec61 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -3,7 +3,7 @@ pip==9.0.1 flake8==3.3.0 pyflakes==1.5.0 coverage==4.4.1 -cchardet==2.1.0 +cchardet==2.1.1 cython==0.25.2 chardet==3.0.4 isort==4.2.15 From 946b772b41a3a8b94b3de991ea2de189584325b8 Mon Sep 17 00:00:00 2001 From: Frederik Aalund Date: Sat, 1 Jul 2017 20:22:03 +0200 Subject: [PATCH 053/167] Add graceful shutdown section to the client usage documentation (#2045) * Docs: Added graceful shutdown procedure to client * Changes: Create 2039.doc --- changes/2039.doc | 1 + docs/client.rst | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 changes/2039.doc diff --git a/changes/2039.doc b/changes/2039.doc new file mode 100644 index 00000000000..e08b12deef7 --- /dev/null +++ b/changes/2039.doc @@ -0,0 +1 @@ +Add a graceful shutdown section to the client usage documentation. diff --git a/docs/client.rst b/docs/client.rst index 76074bfd0b9..39f6e94ce06 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -739,3 +739,35 @@ reading procedures:: Timeout is cumulative time, it includes all operations like sending request, redirects, response parsing, consuming response, etc. + + +Graceful Shutdown +----------------- + +When ``ClientSession`` closes at the end of an ``async with`` block (or through a direct ``.close()`` call), the underlying connection remains open due to asyncio internal details. In practice, the underlying connection will close after a short while. However, if the event loop is stopped before the underlying connection is closed, an ``ResourceWarning: unclosed transport`` warning is emitted (when warnings are enabled). + +To avoid this situation, a small delay must be added before closing the event loop to allow any open underlying connections to close. + +For a ``ClientSession`` without SSL, a simple zero-sleep (``await asyncio.sleep(0)``) will suffice:: + + async def read_website(): + async with aiohttp.ClientSession() as session: + async with session.get('http://example.org/') as response: + await response.read() + + loop = asyncio.get_event_loop() + loop.run_until_complete(read_website()) + # Zero-sleep to allow underlying connections to close + loop.run_until_complete(asyncio.sleep(0)) + loop.close() + +For a ``ClientSession`` with SSL, the application must wait a short duration before closing:: + + ... + # Wait 250 ms for the underlying SSL connections to close + loop.run_until_complete(asyncio.sleep(0.250)) + loop.close() + +Note that the appropriate amount of time to wait will vary from application to application. + +All if this will eventually become obsolete when the asyncio internals are changed so that aiohttp itself can wait on the underlying connection to close. Please follow issue `#1925 `_ for the progress on this. From 2ae4540de5bd65f1ef2668467588af6cc9134d53 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sun, 2 Jul 2017 16:28:59 +0200 Subject: [PATCH 054/167] Update sphinx from 1.6.2 to 1.6.3 (#2047) --- requirements/doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/doc.txt b/requirements/doc.txt index 9f10577695e..7b908f59f81 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,3 +1,3 @@ -sphinx==1.6.2 +sphinx==1.6.3 sphinxcontrib-asyncio==0.2.0 pygments>=2.1 From cc9f40998a4419968fc98e54d916c82ef98fbc56 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Sun, 2 Jul 2017 13:45:32 -0700 Subject: [PATCH 055/167] Yield from session.close() to silence warnings (#2049) --- aiohttp/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 8b55c1fdd50..5f8cce3cefe 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -330,7 +330,7 @@ def close(self): resp.close() for ws in self._websockets: yield from ws.close() - self._session.close() + yield from self._session.close() yield from self._server.close() self._closed = True From ed0e85b0b87cedf41d505053ecb3016f449a56fb Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Mon, 3 Jul 2017 17:01:31 +0200 Subject: [PATCH 056/167] Add Skyscanner Hotels as site powered by aiohttp (#2050) * Add Skyscanner Hotels as site powered by aiohttp * Updated spelling wordlist --- docs/powered_by.rst | 1 + docs/spelling_wordlist.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index f2e52ac7d2d..0bf146c9f87 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -11,3 +11,4 @@ make a Pull Request! * https://www.farmersbusinessnetwork.com * `Home Assistant `_ * KeepSafe https://www.getkeepsafe.com/ +* Skyscanner Hotels https://www.skyscanner.net/hotels diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 7bac57a06e5..f8891b9efd1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -242,3 +242,4 @@ WSMessage WSMsgType wss www +Skyscanner From fd71bc27ba15901953ea10d46a3bc5261da279a1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 3 Jul 2017 18:02:26 +0300 Subject: [PATCH 057/167] Get rid of custom URL, use yarl.URL.build instead (#2051) * Get rid of custom URL, use yarl.URL.build instead * Add missing changes record --- aiohttp/_http_parser.pyx | 14 ++++++++++---- aiohttp/http_parser.py | 4 ++-- aiohttp/http_writer.py | 32 -------------------------------- changes/2048.misc | 1 + 4 files changed, 13 insertions(+), 38 deletions(-) create mode 100644 changes/2048.misc diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 64f6d0dd632..4af5d20d21f 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -7,14 +7,14 @@ from cpython.mem cimport PyMem_Malloc, PyMem_Free from cpython cimport PyObject_GetBuffer, PyBuffer_Release, PyBUF_SIMPLE, \ Py_buffer, PyBytes_AsString -import yarl from multidict import CIMultiDict +from yarl import URL from aiohttp import hdrs from .http_exceptions import ( BadHttpMessage, BadStatusLine, InvalidHeader, LineTooLong, InvalidURLError, PayloadEncodingError, ContentLengthError, TransferEncodingError) -from .http_writer import HttpVersion, HttpVersion10, HttpVersion11, URL +from .http_writer import HttpVersion, HttpVersion10, HttpVersion11 from .http_parser import RawRequestMessage, RawResponseMessage, DeflateBuffer from .streams import EMPTY_PAYLOAD, FlowControlStreamReader @@ -329,7 +329,7 @@ cdef int cb_on_url(cparser.http_parser* parser, pyparser._path = path if pyparser._cparser.method == 5: # CONNECT - pyparser._url = yarl.URL(path) + pyparser._url = URL(path) else: pyparser._url = _parse_url(at[:length], length) except BaseException as ex: @@ -486,6 +486,8 @@ def _parse_url(char* buf_data, size_t length): str path = None str query = None str fragment = None + str user = None + str password = None str userinfo = None object result = None int off @@ -541,7 +543,11 @@ def _parse_url(char* buf_data, size_t length): ln = parsed.field_data[cparser.UF_USERINFO].len userinfo = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - return URL(schema, host, port, path, query, fragment, userinfo) + user, sep, password = userinfo.partition(':') + + return URL.build(scheme=schema, + user=user, password=password, host=host, port=port, + path=path, query=query, fragment=fragment) else: raise InvalidURLError("invalid url {!r}".format(buf_data)) finally: diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 15df56981fd..c2fb6819229 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -4,8 +4,8 @@ import zlib from enum import IntEnum -import yarl from multidict import CIMultiDict +from yarl import URL from . import hdrs from .helpers import NO_EXTENSIONS @@ -375,7 +375,7 @@ def parse_message(self, lines): return RawRequestMessage( method, path, version, headers, raw_headers, - close, compression, upgrade, chunked, yarl.URL(path)) + close, compression, upgrade, chunked, URL(path)) class HttpResponseParserPy(HttpParser): diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 49ce32a30c6..ae3d6c9d3eb 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -4,9 +4,6 @@ import collections import socket import zlib -from urllib.parse import SplitResult - -import yarl from .abc import AbstractPayloadWriter from .helpers import create_future, noop @@ -299,32 +296,3 @@ def drain(self, last=False): self._drain_waiter = create_future(self.loop) yield from self._drain_waiter - - -class URL(yarl.URL): - - def __init__(self, schema, netloc, port, path, query, fragment, userinfo): - self._strict = False - - if port: - netloc += ':{}'.format(port) - if userinfo: - netloc = yarl.quote( - userinfo, safe='@:', - protected=':', strict=False) + '@' + netloc - - if path: - path = yarl.quote(path, safe='@:', protected='/', strict=False) - - if query: - query = yarl.quote( - query, safe='=+&?/:@', - protected=yarl.PROTECT_CHARS, qs=True, strict=False) - - if fragment: - fragment = yarl.quote(fragment, safe='?/:@', strict=False) - - self._val = SplitResult( - schema or '', # scheme - netloc=netloc, path=path, query=query, fragment=fragment) - self._cache = {} diff --git a/changes/2048.misc b/changes/2048.misc new file mode 100644 index 00000000000..def1e4c551f --- /dev/null +++ b/changes/2048.misc @@ -0,0 +1 @@ +Replace custom URL with `yarl.URL.build` From a598b1ac1fd7cc0321a7ea17cdeee897487c6e1e Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sun, 9 Jul 2017 00:21:34 +0200 Subject: [PATCH 058/167] Update pytest to 3.1.3 (#2054) * Update pytest from 3.1.2 to 3.1.3 * Update pytest from 3.1.2 to 3.1.3 --- requirements/ci.txt | 2 +- requirements/wheel.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 327ae80ec61..dc55f983cea 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -10,7 +10,7 @@ isort==4.2.15 tox==2.7.0 multidict==3.1.0 async-timeout==1.2.1 -pytest==3.1.2 +pytest==3.1.3 pytest-cov==2.5.1 pytest-mock==1.6.0 gunicorn==19.7.1 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index ddcecc7cd8c..5dba562b955 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ cython==0.25.2 -pytest==3.1.2 +pytest==3.1.3 twine==1.9.1 From 11004ff307f95c85d293d49b4d4dd6e39143c688 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sun, 9 Jul 2017 00:42:23 +0200 Subject: [PATCH 059/167] Update multidict from 3.1.0 to 3.1.1 (#2071) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index dc55f983cea..9bb3f7c058e 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ cython==0.25.2 chardet==3.0.4 isort==4.2.15 tox==2.7.0 -multidict==3.1.0 +multidict==3.1.1 async-timeout==1.2.1 pytest==3.1.3 pytest-cov==2.5.1 From e1dbb98e98e8019e295958a4332c6d79e84e539c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jul 2017 09:16:03 +0200 Subject: [PATCH 060/167] Add ocean.io to powered by list --- docs/powered_by.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index 0bf146c9f87..4f74b50fc26 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -12,3 +12,4 @@ make a Pull Request! * `Home Assistant `_ * KeepSafe https://www.getkeepsafe.com/ * Skyscanner Hotels https://www.skyscanner.net/hotels +* Ocean S.A. https://ocean.io/ From 4ecb6339ddb60ad732e8a79a32ce75c10ec8546d Mon Sep 17 00:00:00 2001 From: Andrew Lytvyn Date: Mon, 10 Jul 2017 12:04:12 +0300 Subject: [PATCH 061/167] ClientSession.__aexit__ returns ClientSession.close coroutine (#2064) * `yield from session.close()` for asynchronous context manager ClientSession * ClientSession.__aexit__ returns close() coroutine instead of just calling * remove redundant coroutine for __axit__ * add news fragment to changes folder * yield from session.close() in ClientSession.__aexit__ coroutine function --- aiohttp/client.py | 2 +- changes/2063.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/2063.bugfix diff --git a/aiohttp/client.py b/aiohttp/client.py index 3b33fc48ee3..4f3dad59de8 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -584,7 +584,7 @@ def __aenter__(self): @asyncio.coroutine def __aexit__(self, exc_type, exc_val, exc_tb): - self.close() + yield from self.close() class _BaseRequestContextManager(_BaseCoroMixin): diff --git a/changes/2063.bugfix b/changes/2063.bugfix new file mode 100644 index 00000000000..0bdfdc20ac5 --- /dev/null +++ b/changes/2063.bugfix @@ -0,0 +1 @@ +Fix issue with synchronous session closing when using ClientSession as an asynchronous context manager. \ No newline at end of file From a57797c8318b82a2bc67f871ae30384d317ef072 Mon Sep 17 00:00:00 2001 From: cockcrow Date: Tue, 11 Jul 2017 20:10:24 +0800 Subject: [PATCH 062/167] Fix docs (#2078) `CIMultiDict` typo and links --- docs/client.rst | 2 +- docs/client_reference.rst | 4 ++-- docs/web_reference.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index 39f6e94ce06..dedaf51d5d4 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -604,7 +604,7 @@ Response Headers ---------------- We can view the server's response :attr:`ClientResponse.headers` using -a :class:`CIMultiDictProxy`:: +a :class:`~multidict.CIMultiDictProxy`:: >>> resp.headers {'ACCESS-CONTROL-ALLOW-ORIGIN': '*', diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 71c8d52b1e9..17bb4c6c732 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -73,7 +73,7 @@ The client session supports the context manager protocol for self closing. May be either *iterable of key-value pairs* or :class:`~collections.abc.Mapping` (e.g. :class:`dict`, - :class:`~aiohttp.CIMultiDict`). + :class:`~multidict.CIMultiDict`). :param skip_auto_headers: set of headers for which autogeneration should be skipped. @@ -997,7 +997,7 @@ Response object .. attribute:: headers A case-insensitive multidict proxy with HTTP headers of - response, :class:`CIMultiDictProxy`. + response, :class:`~multidict.CIMultiDictProxy`. .. attribute:: raw_headers diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 8e2b5696bbe..96752acfe3a 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -290,7 +290,7 @@ and :ref:`aiohttp-web-signals` handlers. :param rel_url: url to use, :class:`str` or :class:`~yarl.URL` - :param headers: :class:`~multidict.CIMultidict` or compatible + :param headers: :class:`~multidict.CIMultiDict` or compatible headers container. :return: a cloned :class:`Request` instance. @@ -569,7 +569,7 @@ StreamResponse .. attribute:: headers - :class:`~aiohttp.CIMultiiDct` instance + :class:`~multidict.CIMultiDict` instance for *outgoing* *HTTP headers*. .. attribute:: cookies From 8a42d2a831118b59c771eb68a8ead463bb7a4aa3 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 12 Jul 2017 17:22:06 +0200 Subject: [PATCH 063/167] Add GNS3 to powered by (#2080) --- docs/powered_by.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index 4f74b50fc26..74d32ddc2e3 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -13,3 +13,4 @@ make a Pull Request! * KeepSafe https://www.getkeepsafe.com/ * Skyscanner Hotels https://www.skyscanner.net/hotels * Ocean S.A. https://ocean.io/ +* GNS3 http://gns3.com From 46e7a2f350c6dd1b0b28baac6ed3c9f6afba0c76 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 12 Jul 2017 17:22:34 +0200 Subject: [PATCH 064/167] Fix spelling to satisfy docs linter (#2082) * Also whitelist certain words --- docs/built_with.rst | 2 +- docs/client_reference.rst | 2 +- docs/spelling_wordlist.txt | 15 +++++++++++++++ docs/third_party.rst | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/built_with.rst b/docs/built_with.rst index c1bb49dce18..aeaac9c5664 100644 --- a/docs/built_with.rst +++ b/docs/built_with.rst @@ -7,7 +7,7 @@ aiohttp is used to build useful libraries built on top of it, and there's a page dedicated to list them: :ref:`aiohttp-3rd-party`. There are also projects that leverage the power of aiohttp to -provide end-user tools, like command lines or softwares with +provide end-user tools, like command lines or software with full user interfaces. This page aims to list those projects. If you are using aiohttp diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 17bb4c6c732..c7925537d62 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -986,7 +986,7 @@ Response object Reading from the stream may raise :exc:`aiohttp.ClientPayloadError` if the response object is closed before response receives all data or in case if any - transfer encoding related errors like mis-formed chunked + transfer encoding related errors like misformed chunked encoding of broken compression data. .. attribute:: cookies diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index f8891b9efd1..dadca29248a 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -32,6 +32,7 @@ Changelog charset charsetdetect chunked +chunking CIMultiDict ClientSession cls @@ -49,10 +50,13 @@ cpu css ctor Ctrl +Cython cythonized de deduplicate +deprecations DER +Dev dict Dict django @@ -67,6 +71,8 @@ environ eof epoll Facebook +# de-facto: +facto fallback filename finalizers @@ -90,6 +96,7 @@ incapsulates Indices infos inline +intaking io ip IP @@ -118,6 +125,8 @@ metadata middleware middlewares miltidict +misbehaviors +misformed Mongo msg MsgType @@ -145,6 +154,7 @@ param params pathlib ping +pipelining pluggable plugin poller @@ -157,11 +167,13 @@ PRs pubsub py pyenv +pyflakes pytest Pytest readonly readpayload rebase +redirections Redis refactor Refactor @@ -170,6 +182,7 @@ refactoring regex regexps regexs +reloader renderer renderers repo @@ -180,6 +193,7 @@ resolvers reusage Runit sa +Satisfiable schemas sendfile serializable @@ -188,6 +202,7 @@ skipuntil SocketSocketTransport ssl SSLContext +startup subclasses submodules subpackage diff --git a/docs/third_party.rst b/docs/third_party.rst index 3483710769c..7960c3a7631 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -99,7 +99,7 @@ Database drivers Others ------ -The list of libs which are exists but not enlisted in former categories. +The list of libraries which are exists but not enlisted in former categories. They are may be perfect or not -- we don't know. From b14b01479b159def2d260e81c05bb40d837abd92 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Wed, 12 Jul 2017 17:29:26 +0200 Subject: [PATCH 065/167] Update the doc of web.Application (#2081) --- changes/2081.doc | 1 + docs/web_reference.rst | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 changes/2081.doc diff --git a/changes/2081.doc b/changes/2081.doc new file mode 100644 index 00000000000..a8dd55860b7 --- /dev/null +++ b/changes/2081.doc @@ -0,0 +1 @@ +Update the doc of web.Application diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 96752acfe3a..b4725eb3b37 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1157,23 +1157,36 @@ properties for later access from a :ref:`handler` via the Although :class:`Application` is a :obj:`dict`-like object, it can't be duplicated like one using :meth:`Application.copy`. -.. class:: Application(*, router=None, logger=, \ - middlewares=(), debug=False, **kwargs) +.. class:: Application(*, logger=, router=None,middlewares=(), \ + handler_args=None, client_max_size=1024**2, \ + secure_proxy_ssl_header=None, loop=None, debug=...) The class inherits :class:`dict`. - :param router: :class:`aiohttp.abc.AbstractRouter` instance, the system - creates :class:`UrlDispatcher` by default if - *router* is ``None``. - :param logger: :class:`logging.Logger` instance for storing application logs. By default the value is ``logging.getLogger("aiohttp.web")`` + :param router: :class:`aiohttp.abc.AbstractRouter` instance, the system + creates :class:`UrlDispatcher` by default if + *router* is ``None``. + :param middlewares: :class:`list` of middleware factories, see :ref:`aiohttp-web-middlewares` for details. - :param debug: Switches debug mode. + :param handler_args: dict-like object that overrides keyword arguments of + :meth:`Application.make_handler` + + :param client_max_size: client's maximum size in a request. If a POST + request exceeds this value, it raises an + `HTTPRequestEntityTooLarge` exception. + + :param tuple secure_proxy_ssl_header: Default: ``None``. + + .. deprecated:: 2.1 + + See ``request.url.scheme`` for built-in resolution of the current + scheme using the standard and de-facto standard headers. :param loop: event loop @@ -1182,6 +1195,8 @@ duplicated like one using :meth:`Application.copy`. The parameter is deprecated. Loop is get set during freeze stage. + :param debug: Switches debug mode. + .. attribute:: router Read-only property that returns *router instance*. From dad8ef7a63d01b06b0b8fa54d56269df969c1932 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 12 Jul 2017 20:16:03 +0100 Subject: [PATCH 066/167] add TutorCruncher Socket and Morpheus to powered by (#2083) --- docs/powered_by.rst | 2 ++ docs/spelling_wordlist.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index 74d32ddc2e3..8b99d16c763 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -14,3 +14,5 @@ make a Pull Request! * Skyscanner Hotels https://www.skyscanner.net/hotels * Ocean S.A. https://ocean.io/ * GNS3 http://gns3.com +* TutorCruncher socket https://tutorcruncher.com/features/tutorcruncher-socket/ +* Morpheus messaging microservice https://github.com/tutorcruncher/morpheus diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index dadca29248a..24c9f9beb37 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -122,6 +122,7 @@ lookups Mako manylinux metadata +microservice middleware middlewares miltidict From dfb9e1d0bbf037f4316fb2078bf03870061c6b63 Mon Sep 17 00:00:00 2001 From: Nikolay Novik Date: Thu, 13 Jul 2017 00:17:40 +0300 Subject: [PATCH 067/167] Update README.rst (#2079) --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 31662592df3..2d58ba63c29 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,9 @@ Async http client/server framework .. image:: https://badge.fury.io/py/aiohttp.svg :target: https://badge.fury.io/py/aiohttp +.. image:: https://badges.gitter.im/Join%20Chat.svg + :target: https://gitter.im/aio-libs/Lobby + :alt: Chat on Gitter aiohttp 2.0 release! ==================== From b0a2600320018d62daba8a5e2eff141b7ac39ec2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 13 Jul 2017 10:54:25 +0200 Subject: [PATCH 068/167] Mention Discord --- docs/third_party.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/third_party.rst b/docs/third_party.rst index 7960c3a7631..fd3109fbc10 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -194,3 +194,5 @@ period ask to raise he status. - `home-assistant `_ Open-source home automation platform running on Python 3. + +- `https://github.com/Rapptz/discord.py`_ Discord client library. From ffd98ada3cef8f99624c7351c01fa3c127fab994 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 13 Jul 2017 11:09:52 +0200 Subject: [PATCH 069/167] Fix spelling --- docs/spelling_wordlist.txt | 1 + docs/third_party.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 24c9f9beb37..71261b854d4 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -59,6 +59,7 @@ DER Dev dict Dict +Discord django Django dns diff --git a/docs/third_party.rst b/docs/third_party.rst index fd3109fbc10..1b53c7856f3 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -195,4 +195,4 @@ period ask to raise he status. - `home-assistant `_ Open-source home automation platform running on Python 3. -- `https://github.com/Rapptz/discord.py`_ Discord client library. +- `discord.py `_ Discord client library. From cbb571076512d1844c9af95c5e63e62ecdd46599 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 13 Jul 2017 11:11:54 +0200 Subject: [PATCH 070/167] Sort spelling --- docs/spelling_wordlist.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 71261b854d4..46e2a0a47ab 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -16,10 +16,10 @@ autodetection autogenerates autogeneration awaitable -Backport -BaseEventLoop backend backends +Backport +BaseEventLoop basename BasicAuth BodyPartReader @@ -54,6 +54,7 @@ Cython cythonized de deduplicate +# de-facto: deprecations DER Dev @@ -72,7 +73,6 @@ environ eof epoll Facebook -# de-facto: facto fallback filename @@ -201,6 +201,7 @@ sendfile serializable shourtcuts skipuntil +Skyscanner SocketSocketTransport ssl SSLContext @@ -259,4 +260,3 @@ WSMessage WSMsgType wss www -Skyscanner From 351690505a332d1240442860c5ec37b67aa475f6 Mon Sep 17 00:00:00 2001 From: Wang Feng Date: Thu, 13 Jul 2017 20:18:07 +0900 Subject: [PATCH 071/167] Fix docs (#2085) (#2086) mistake about disabling access logs. --- changes/2085.doc | 1 + docs/logging.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/2085.doc diff --git a/changes/2085.doc b/changes/2085.doc new file mode 100644 index 00000000000..247c6fcc277 --- /dev/null +++ b/changes/2085.doc @@ -0,0 +1 @@ +Fix mistake about access log disabling. diff --git a/docs/logging.rst b/docs/logging.rst index 3a0686e56cd..ed1f6819be5 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -38,7 +38,7 @@ instance to override default logger. .. note:: - Use ``app.make_handler(access_log=None)`` for disabling access logs. + Use ``web.run_app(app, access_log=None)`` for disabling access logs. Other parameter called *access_log_format* may be used for specifying log From fb0a7814b53d990493540d0d4ed34395ad5e019b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 13 Jul 2017 23:37:10 +0200 Subject: [PATCH 072/167] Mention Arsenic in docs --- docs/built_with.rst | 1 + docs/spelling_wordlist.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/built_with.rst b/docs/built_with.rst index aeaac9c5664..8fb38c87767 100644 --- a/docs/built_with.rst +++ b/docs/built_with.rst @@ -19,3 +19,4 @@ project, pointing to ``_. * `Molotov `_ Load testing tool. +* `Arsenic `_ Async WebDriver. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 46e2a0a47ab..d2b376190cd 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -8,6 +8,7 @@ alives api app arg +Arsenic async asyncio auth From eccb368a5f7fb91cf37b4009c4959711a38b77ab Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Fri, 14 Jul 2017 12:40:22 +0200 Subject: [PATCH 073/167] Update yarl from 0.11.0 to 0.12.0 (#2091) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 9bb3f7c058e..9a746d7c64a 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -16,5 +16,5 @@ pytest-mock==1.6.0 gunicorn==19.7.1 #aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 -yarl==0.11.0 +yarl==0.12.0 -e . From d6258e7b7a26a7caf6e6644e390d685ddfb503f5 Mon Sep 17 00:00:00 2001 From: Ludovic Gasc Date: Fri, 14 Jul 2017 23:48:40 +0200 Subject: [PATCH 074/167] Add Eyepea + ALLOcloud links and better links rendering (#2097) --- docs/powered_by.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index 8b99d16c763..c1daac4fade 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -8,11 +8,13 @@ Web sites powered by aiohttp. Feel free to fork documentation on github, add a link to your site and make a Pull Request! -* https://www.farmersbusinessnetwork.com +* `Farmer Business Network `_ * `Home Assistant `_ -* KeepSafe https://www.getkeepsafe.com/ -* Skyscanner Hotels https://www.skyscanner.net/hotels -* Ocean S.A. https://ocean.io/ -* GNS3 http://gns3.com -* TutorCruncher socket https://tutorcruncher.com/features/tutorcruncher-socket/ -* Morpheus messaging microservice https://github.com/tutorcruncher/morpheus +* `KeepSafe `_ +* `Skyscanner Hotels `_ +* `Ocean S.A. `_ +* `GNS3 `_ +* `TutorCruncher socket `_ +* `Morpheus messaging microservice `_ +* `Eyepea - Custom telephony solutions `_ +* `ALLOcloud - Telephony in the cloud `_ From df19ad4acba6e538d36861034f8854fb6558fe6c Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sat, 15 Jul 2017 09:16:24 +0200 Subject: [PATCH 075/167] Update multidict from 3.1.1 to 3.1.3 (#2093) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 9a746d7c64a..56b71d113b7 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ cython==0.25.2 chardet==3.0.4 isort==4.2.15 tox==2.7.0 -multidict==3.1.1 +multidict==3.1.3 async-timeout==1.2.1 pytest==3.1.3 pytest-cov==2.5.1 From 33e883c248d6de46778f6f72169e12303a2f3151 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 15 Jul 2017 09:49:17 +0200 Subject: [PATCH 076/167] Rename *Client* doc's chapter to *Client Usage* --- docs/client.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index dedaf51d5d4..047de14b9cc 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -1,7 +1,7 @@ .. _aiohttp-client: -Client -====== +Client Usage +============ .. module:: aiohttp From f36bdbce2bedc808464aa025ade35015027b775f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 15 Jul 2017 16:06:58 +0200 Subject: [PATCH 077/167] prevent codecov blames about reducing total lines in code, check PR only --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codecov.yml b/codecov.yml index 543a9f13646..4487fb3690f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,2 +1,5 @@ coverage: range: "95..100" + + status: + project: no From 334731e27efef8ac04d6851f60fe93368212fc27 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 4 Jun 2017 01:20:15 +0300 Subject: [PATCH 078/167] Use pipelines in Travis CI --- .travis.yml | 69 +++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4aba6a9a40d..d621887853a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,36 +4,48 @@ services: language: python -python: - # python3.4.2 has bug in http.cookies module, aiohttp provides fix - - 3.4.2 - - 3.4.3 - - 3.5.2 - # - 3.5.3 - - 3.5 - - 3.6 - # - 3.7-dev - # - 'nightly' +addons: + apt: + packages: + - enchant -os: - - linux -# - osx # doesn't work on MacOSX -- the system has no Python installed +# doesn't work on MacOSX out of the box -- the system has no Python installed +# there's a workaround to use `language: generic` and install it, but it's slow +os: linux -matrix: +jobs: allow_failures: - - python: '3.6-dev' - - python: 'nightly' - - os: osx + - python: 3.6-dev + - python: nightly -cache: - directories: - - $HOME/.cache/pip + include: + # python3.4.2 has bug in http.cookies module, aiohttp provides fix + - python: 3.4.2 + - python: 3.4.3 + - python: 3.5.2 + # - 3.5.3 + - python: 3.5 + - python: &mainstream_python 3.6 + - python: 3.6-dev + - python: nightly -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log + - stage: deploy (PYPI upload itself runs only for tagged commits) + python: *mainstream_python + deploy: + provider: pypi + user: andrew.svetlov + password: + secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= + distributions: sdist + on: + tags: true + all_branches: true + +cache: pip + +before_cache: rm -f $HOME/.cache/pip/log/debug.log install: - - sudo apt-get install enchant - pip install --upgrade pip wheel - pip install --upgrade setuptools - pip install --upgrade setuptools-git @@ -56,14 +68,3 @@ script: after_success: - codecov - ./tools/run_docker.sh - -deploy: - provider: pypi - user: andrew.svetlov - password: - secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= - distributions: "sdist" - on: - tags: true - all_branches: true - python: 3.6 From 6b1fa84bf60cf91ecd95d420f0d486f942c872e7 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 4 Jun 2017 01:22:52 +0300 Subject: [PATCH 079/167] Don't run test-related stages in deploy job Closes #1926 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d621887853a..d0a4a6489e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + install: skip + script: skip + after_success: skip deploy: provider: pypi user: andrew.svetlov From 263c43ab98c098acfcc97ef4283f59f5852a94fb Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:19:58 +0300 Subject: [PATCH 080/167] Use environmental marker for uvloop install --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0a4a6489e4..82032d71fab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,7 @@ install: - pip install -r requirements/ci.txt - pip install aiodns - pip install codecov - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - pip install uvloop; - fi + - pip install 'uvloop; python_version>="3.5"' - pip install sphinxcontrib-spelling script: From 5f77bf190fb076b3c8beee7a2709a112a5ee7a77 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:21:04 +0300 Subject: [PATCH 081/167] Optimize docker running script * Introduce package_name script param * Reduce nesting * Reduce code duplication --- .travis.yml | 2 +- tools/run_docker.sh | 46 ++++++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82032d71fab..33fd740f978 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,4 +68,4 @@ script: after_success: - codecov - - ./tools/run_docker.sh + - ./tools/run_docker.sh "aiohttp" diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 056eb51d583..7d0bfbf2e36 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,25 +1,33 @@ -if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENSIONS] ;then - echo "x86_64" - docker pull quay.io/pypa/manylinux1_x86_64 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/tools/build-wheels.sh - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_x86_64.whl - do - echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD - done - echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 rm -rf /io/dist +#!/bin/bash +if [ -z $TRAVIS_TAG ] || [ ! -z $PYTHONASYNCIODEBUG ] || [ ! -z $AIOHTTP_NO_EXTENSIONS ] +then + exit 1 +fi + +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 +fi + +dock_ext_args="" + +for arch in x86_64 i686 +do + [ $arch == "i686" ] && dock_ext_args="linux32" + + echo "${arch}" + docker pull "quay.io/pypa/manylinux1_${arch}" + docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" - echo "i686" - docker pull quay.io/pypa/manylinux1_i686 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/tools/build-wheels.sh echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_i686.whl + for f in dist/aiohttp*manylinux1_${arch}.whl do echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD + python -m twine upload "$f" --username andrew.svetlov --password "$PYPI_PASSWD" done + echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 rm -rf /io/dist -fi + docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" rm -rf /io/dist +done From 0bec60ea49a8ecff025eda65b36d5869bee9c39e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:23:20 +0300 Subject: [PATCH 082/167] Introduce package_name param @ build wheels script --- tools/build-wheels.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 4c2f2724daa..cff482c2321 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,6 +1,13 @@ #!/bin/bash PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 +fi + echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt @@ -8,13 +15,13 @@ for PYTHON in ${PYTHON_VERSIONS}; do done echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/aiohttp*.whl; do - auditwheel repair $whl -w /io/dist/ +for whl in /io/dist/${package_name}*.whl; do + auditwheel repair "$whl" -w /io/dist/ done echo "Install packages and test" for PYTHON in ${PYTHON_VERSIONS}; do - /opt/python/${PYTHON}/bin/pip install aiohttp --no-index -f file:///io/dist + /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist rm -rf /io/tests/__pycache__ rm -rf /io/tests/test_py35/__pycache__ /opt/python/${PYTHON}/bin/py.test /io/tests From ee042d481a74c5f2ea9d039b51ecfadc18025dab Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:23:54 +0300 Subject: [PATCH 083/167] Avoid pycache with PYTHONDONTWRITEBYTECODE env var --- tools/build-wheels.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index cff482c2321..e9a45065112 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,6 +1,9 @@ #!/bin/bash PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +# Avoid creation of __pycache__/*.py[c|o] +export PYTHONDONTWRITEBYTECODE=1 + package_name="$1" if [ -z "$package_name" ] then @@ -22,9 +25,5 @@ done echo "Install packages and test" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ /opt/python/${PYTHON}/bin/py.test /io/tests - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ done From 7eb5a9e710edd1375323d38d92bdbddff92997f6 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:56:16 +0300 Subject: [PATCH 084/167] Fail fast in Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 33fd740f978..e15fa53d5b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ addons: os: linux jobs: + fast_finish: true allow_failures: - python: 3.6-dev - python: nightly From 4b390a8500d86d44c7b1cb89b0faeeaa5b4b2695 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:13:56 +0300 Subject: [PATCH 085/167] Clean up dist folder in host system Motivation: eliminate container spawn overhead --- tools/run_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 7d0bfbf2e36..0fc3faa9fdf 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -29,5 +29,5 @@ do done echo "Cleanup" - docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" rm -rf /io/dist + rm -rf ./dist done From e24369d57df86c4cb3e90a68d8751071c3f0ae90 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:21:17 +0300 Subject: [PATCH 086/167] Use simplier check flags in bash script --- tools/run_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 0fc3faa9fdf..453c0c468af 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,5 +1,5 @@ #!/bin/bash -if [ -z $TRAVIS_TAG ] || [ ! -z $PYTHONASYNCIODEBUG ] || [ ! -z $AIOHTTP_NO_EXTENSIONS ] +if [ -z $TRAVIS_TAG ] || [ -n $PYTHONASYNCIODEBUG ] || [ -n $AIOHTTP_NO_EXTENSIONS ] then exit 1 fi From 68cd3b678fccc27ff1524c43d5787661a9a78ccd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:46:00 +0300 Subject: [PATCH 087/167] Don't require VM in Travis CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e15fa53d5b2..1410f97a167 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: required +sudo: false services: - docker From ad6037e6c3dcbbd93f05b70a1cd278d1e3a640d5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:24:00 +0300 Subject: [PATCH 088/167] Move docker dependency to deploy job only --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1410f97a167..df9b21fce85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ sudo: false -services: - - docker language: python @@ -32,8 +30,11 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + services: + - docker install: skip - script: skip + script: + - ./tools/run_docker.sh "aiohttp" after_success: skip deploy: provider: pypi @@ -69,4 +70,3 @@ script: after_success: - codecov - - ./tools/run_docker.sh "aiohttp" From 8ff0f5e64e3d7142fa3d562fdcb41a4def52aa1c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:26:01 +0300 Subject: [PATCH 089/167] Move mainstream python var to helpers --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df9b21fce85..aaaf62d36f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ sudo: false language: python +_helpers: +- &mainstream_python 3.6 +- &_mainstream_python_base + python: *mainstream_python addons: apt: packages: @@ -24,7 +28,7 @@ jobs: - python: 3.5.2 # - 3.5.3 - python: 3.5 - - python: &mainstream_python 3.6 + - <<: *_mainstream_python_base - python: 3.6-dev - python: nightly From a548ced03ab373d2ba9578b2c1da3c7af344eb0b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:26:55 +0300 Subject: [PATCH 090/167] Move common requirements to requirements/ci.txt --- .travis.yml | 10 ++-------- appveyor.yml | 1 - requirements/ci.txt | 9 ++++++++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index aaaf62d36f1..905edefcf90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,14 +55,8 @@ cache: pip before_cache: rm -f $HOME/.cache/pip/log/debug.log install: - - pip install --upgrade pip wheel - - pip install --upgrade setuptools - - pip install --upgrade setuptools-git - - pip install -r requirements/ci.txt - - pip install aiodns - - pip install codecov - - pip install 'uvloop; python_version>="3.5"' - - pip install sphinxcontrib-spelling + - pip install --upgrade pip setuptools wheel + - pip install -U -r requirements/ci.txt script: - make cov-dev-full diff --git a/appveyor.yml b/appveyor.yml index f6931d03da4..9778a5097f7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,6 @@ environment: install: - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" - - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine" - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt" build: false diff --git a/requirements/ci.txt b/requirements/ci.txt index 56b71d113b7..0259fe916b5 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,3 +1,5 @@ +setuptools-git + -r doc.txt pip==9.0.1 flake8==3.3.0 @@ -14,7 +16,12 @@ pytest==3.1.3 pytest-cov==2.5.1 pytest-mock==1.6.0 gunicorn==19.7.1 -#aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 yarl==0.12.0 -e . + +# Using PEP 508 env markers to control dependency on runtimes: +aiodns; platform_system!="Windows" # required c-ares will not build on windows +codecov; platform_system!="Windows" # We only use it in Travis CI +uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 +sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI From 46d7e3957a669e1b7a45074cb8f0f17f989b047a Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:27:53 +0300 Subject: [PATCH 091/167] Move doc to separate stages, install enchant there --- .travis.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 905edefcf90..486314c08bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,14 @@ _helpers: - &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python -addons: - apt: - packages: - - enchant +- &_doc_base + stage: doc + <<: *_mainstream_python_base + after_failure: cat docs/_build/spelling/output.txt + addons: + apt: + packages: + - enchant # doesn't work on MacOSX out of the box -- the system has no Python installed # there's a workaround to use `language: generic` and install it, but it's slow @@ -32,6 +36,13 @@ jobs: - python: 3.6-dev - python: nightly + - <<: *_doc_base + script: + - make doc + - <<: *_doc_base + script: + - make doc-spelling + - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python services: @@ -61,10 +72,5 @@ install: script: - make cov-dev-full - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - make doc; - make doc-spelling; - fi - after_success: - codecov From 3e160c50ec805649a2b0ccbd718ead9d0092ba90 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:35:49 +0300 Subject: [PATCH 092/167] Install spellcheck requirements only in doc stage --- .travis.yml | 23 +++++++++++++---------- requirements/ci.txt | 1 - requirements/doc-spelling.txt | 1 + 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 requirements/doc-spelling.txt diff --git a/.travis.yml b/.travis.yml index 486314c08bd..fb23214a43a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,16 @@ sudo: false language: python +install: +- &upgrade_python_toolset pip install --upgrade pip setuptools wheel +- pip install -U -r requirements/ci.txt + +script: +- make cov-dev-full + +after_success: +- codecov + _helpers: - &mainstream_python 3.6 - &_mainstream_python_base @@ -9,6 +19,9 @@ _helpers: - &_doc_base stage: doc <<: *_mainstream_python_base + install: + - *upgrade_python_toolset + - pip install -U -r requirements/doc.txt -r requirements/doc-spelling.txt after_failure: cat docs/_build/spelling/output.txt addons: apt: @@ -64,13 +77,3 @@ jobs: cache: pip before_cache: rm -f $HOME/.cache/pip/log/debug.log - -install: - - pip install --upgrade pip setuptools wheel - - pip install -U -r requirements/ci.txt - -script: - - make cov-dev-full - -after_success: - - codecov diff --git a/requirements/ci.txt b/requirements/ci.txt index 0259fe916b5..347a36665db 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -24,4 +24,3 @@ yarl==0.12.0 aiodns; platform_system!="Windows" # required c-ares will not build on windows codecov; platform_system!="Windows" # We only use it in Travis CI uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 -sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt new file mode 100644 index 00000000000..934d9426f0a --- /dev/null +++ b/requirements/doc-spelling.txt @@ -0,0 +1 @@ +sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI From 7e86027f38686ed201c820cb715070b78a7f1ce1 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:49:33 +0300 Subject: [PATCH 093/167] Move aiodns version lock to requirements/ci.txt --- requirements/ci.txt | 2 +- requirements/dev.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 347a36665db..17c739b9bdf 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -21,6 +21,6 @@ yarl==0.12.0 -e . # Using PEP 508 env markers to control dependency on runtimes: -aiodns; platform_system!="Windows" # required c-ares will not build on windows +aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on windows codecov; platform_system!="Windows" # We only use it in Travis CI uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 diff --git a/requirements/dev.txt b/requirements/dev.txt index a92b636d73e..2db07a943dc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,5 +2,4 @@ ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 -aiodns==1.1.1 towncrier==17.4.0 From 3f097e76abeb50af0dbe12c66fd2948037385261 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 18:18:03 +0300 Subject: [PATCH 094/167] Upgrade CI to Ubuntu Trusty --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index fb23214a43a..ab16f6e0d80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ sudo: false +dist: trusty +group: edge + language: python install: From b408d5ab5744452f2e2dca6499af5cc4b8e08128 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 4 Jul 2017 00:25:45 +0300 Subject: [PATCH 095/167] Install dev deps in doc as well --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ab16f6e0d80..30d129ea260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ _helpers: <<: *_mainstream_python_base install: - *upgrade_python_toolset - - pip install -U -r requirements/doc.txt -r requirements/doc-spelling.txt + - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt after_failure: cat docs/_build/spelling/output.txt addons: apt: From d485e67bd79552ca7de1971df74582601660036f Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 12 Jul 2017 19:26:38 +0300 Subject: [PATCH 096/167] Avoid building wheels when not releasing 'em --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 30d129ea260..03043e898a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,15 @@ jobs: - docker install: skip script: - - ./tools/run_docker.sh "aiohttp" + - | + if [ -n "$TRAVIS_TAG" ] + then + echo Proceeding to build wheels + ./tools/run_docker.sh "aiohttp" + else + echo Not building wheels + exit 0 + fi after_success: skip deploy: provider: pypi From cd3261b3396c43358f8d04c50bc2668728b0a50f Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 02:52:22 +0300 Subject: [PATCH 097/167] Skip repo cleanup in deploy step in Travis CI --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03043e898a5..a39ba32d581 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,9 +77,15 @@ jobs: after_success: skip deploy: provider: pypi + # `skip_cleanup: true` is required to preserve binary wheels, built + # inside of manylinux1 docker container during `script` step above + skip_cleanup: true user: andrew.svetlov password: secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= + # Although we instruct `setup.py` to build source distribution, it will + # also upload all wheels we've previously built in manylinux1 docker + # container. distributions: sdist on: tags: true From 7f6af3bc264e2fe9c6d47ec62bd2ea272bf860cb Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 02:55:42 +0300 Subject: [PATCH 098/167] Move group/dist definition into deploy step --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a39ba32d581..0fe839a5e40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ sudo: false -dist: trusty -group: edge - language: python install: @@ -61,6 +58,8 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + dist: trusty + group: edge services: - docker install: skip From 333ed283dafd3cedcdb64255b8f1a5e3844dc4ff Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 08:15:21 +0300 Subject: [PATCH 099/167] Don't proceed doing deploy job stuff if not tagged --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0fe839a5e40..282ed7b1b31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,6 +62,14 @@ jobs: group: edge services: - docker + before_install: + # This must prevent further job progress + - | + if [ -z "$TRAVIS_TAG" ] + then + echo Not building wheels "(before cache)" + exit 0 + fi install: skip script: - | From 3b6e0fef854e82b68744ad4fbb192d41294a3b6d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 10:56:03 +0300 Subject: [PATCH 100/167] Split CI test targets --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1867d66fede..d8766fb088c 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,17 @@ cov-dev: .develop @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests @echo "open file://`pwd`/coverage/index.html" -cov-dev-full: .develop +cov-ci-no-ext: .develop @echo "Run without extensions" @AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests +cov-ci-aio-debug: .develop @echo "Run in debug mode" @PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests +cov-ci-run: .develop @echo "Regular run" @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests + +cov-dev-full: cov-ci-no-ext cov-ci-aio-debug cov-ci-run @echo "open file://`pwd`/coverage/index.html" clean: From 21860391bd0a6cdd6f0f61d7480893511d2c2fb2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 10:58:01 +0300 Subject: [PATCH 101/167] Split Travis CI script step into 3 commands --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 282ed7b1b31..3a571b140fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,9 @@ install: - pip install -U -r requirements/ci.txt script: -- make cov-dev-full +- make cov-ci-no-ext +- make cov-ci-aio-debug +- make cov-ci-run after_success: - codecov From 0f3b22adcfa0c3158a49062ee0648b8c29977c71 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 13:00:50 +0300 Subject: [PATCH 102/167] Optimize Travis CI config --- .travis.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a571b140fa..ac46551c2b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,20 +69,12 @@ jobs: - | if [ -z "$TRAVIS_TAG" ] then - echo Not building wheels "(before cache)" + echo Not building wheels exit 0 fi install: skip script: - - | - if [ -n "$TRAVIS_TAG" ] - then - echo Proceeding to build wheels - ./tools/run_docker.sh "aiohttp" - else - echo Not building wheels - exit 0 - fi + - ./tools/run_docker.sh "aiohttp" after_success: skip deploy: provider: pypi From 725af18a45183a1ce418650674f7f545d057a162 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:00:35 +0300 Subject: [PATCH 103/167] Fix valid manylinux1 wheels build script --- tools/build-wheels.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index e9a45065112..447a62514f1 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -11,19 +11,37 @@ then exit 1 fi +arch=`uname -m` + +echo +echo echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/ done +echo +echo echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/${package_name}*.whl; do +for whl in /io/dist/${package_name}*${arch}.whl; do + echo "Repairing $whl..." auditwheel repair "$whl" -w /io/dist/ done +echo "Cleanup OS specific wheels" +rm -fv /io/dist/*-linux_*.whl + +echo +echo echo "Install packages and test" +echo "dist directory:" +ls /io/dist + for PYTHON in ${PYTHON_VERSIONS}; do + echo + echo -n "Test $PYTHON: " + /opt/python/${PYTHON}/bin/python -c "import platform;print(platform.platform())" /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist /opt/python/${PYTHON}/bin/py.test /io/tests done From cce907a737c6b4c9e66a8e53a4e55a36e083d312 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:01:44 +0300 Subject: [PATCH 104/167] Drop useless env check from docker script --- tools/run_docker.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 453c0c468af..130a6790455 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,9 +1,4 @@ #!/bin/bash -if [ -z $TRAVIS_TAG ] || [ -n $PYTHONASYNCIODEBUG ] || [ -n $AIOHTTP_NO_EXTENSIONS ] -then - exit 1 -fi - package_name="$1" if [ -z "$package_name" ] then From 8854fcb7125650cf33aeb0cfa7da97892893d97e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:02:01 +0300 Subject: [PATCH 105/167] Parallelize pulling manylinux1 docker images --- tools/run_docker.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 130a6790455..1ea8c3a373a 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -6,23 +6,27 @@ then exit 1 fi +manylinux1_image_prefix="quay.io/pypa/manylinux1_" dock_ext_args="" +declare -A docker_pull_pids=() # This syntax requires at least bash v4 for arch in x86_64 i686 do - [ $arch == "i686" ] && dock_ext_args="linux32" + docker pull "${manylinux1_image_prefix}${arch}" & + docker_pull_pids[$arch]=$! +done - echo "${arch}" - docker pull "quay.io/pypa/manylinux1_${arch}" - docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" +for arch in x86_64 i686 +do + echo + echo + arch_pull_pid=${docker_pull_pids[$arch]} + echo waiting for docker pull pid $arch_pull_pid to complete downloading container for $arch arch... + wait $arch_pull_pid # await for docker image for current arch to be pulled from hub + [ $arch == "i686" ] && dock_ext_args="linux32" - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_${arch}.whl - do - echo "Upload $f" - python -m twine upload "$f" --username andrew.svetlov --password "$PYPI_PASSWD" - done + echo Building wheel for $arch arch + docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" - echo "Cleanup" - rm -rf ./dist + dock_ext_args="" # Reset docker args, just in case done From bda81ef0b0c1bd5ae26afce03e2e833bf4adbf88 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:03:36 +0300 Subject: [PATCH 106/167] Add comments explaining Travis deploy section --- .travis.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac46551c2b5..024ab68668b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,15 +79,18 @@ jobs: deploy: provider: pypi # `skip_cleanup: true` is required to preserve binary wheels, built - # inside of manylinux1 docker container during `script` step above + # inside of manylinux1 docker container during `script` step above. skip_cleanup: true user: andrew.svetlov password: secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= - # Although we instruct `setup.py` to build source distribution, it will - # also upload all wheels we've previously built in manylinux1 docker - # container. - distributions: sdist + # Although Travis CI instructs `setup.py` to build source distribution, + # which is default value for distribution option (`distribution: sdist`), + # it will also upload all wheels we've previously built in manylinux1 + # docker container using `twine upload -r pypi dist/*` command. + # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39 + # it is default that Travis PYPI provider has `skip_upload_docs: true` + # set by default. on: tags: true all_branches: true From 70e134f972f2816007b1a8774d845a6f77cb2960 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:09:57 +0300 Subject: [PATCH 107/167] Port setup checks from multidict repo --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 024ab68668b..5b1d6613daa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ script: - make cov-ci-no-ext - make cov-ci-aio-debug - make cov-ci-run +- python setup.py check -rm +- if python -c "import sys; sys.exit(sys.version_info < (3,6))"; then + python setup.py check -s; + fi after_success: - codecov From 013d3eee41b6d1a782ab7bffec9293a26adfeb0c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:10:49 +0300 Subject: [PATCH 108/167] Install just libenchant-dev for doc spellchecker --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5b1d6613daa..75101e36deb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ _helpers: addons: apt: packages: - - enchant + - libenchant-dev # doesn't work on MacOSX out of the box -- the system has no Python installed # there's a workaround to use `language: generic` and install it, but it's slow From 5a77da6895321146dd892cbf3515c0d049e7fc1c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:11:41 +0300 Subject: [PATCH 109/167] Drop unnecessary html doc build stage --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75101e36deb..8336e8e6da3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,6 @@ jobs: - python: 3.6-dev - python: nightly - - <<: *_doc_base - script: - - make doc - <<: *_doc_base script: - make doc-spelling From 2b6377d07ef2851f914c8985e010ebba5c036f6d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:12:35 +0300 Subject: [PATCH 110/167] Add comment about dist deletion in PYPI deployer --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8336e8e6da3..ea8104b7d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,6 +92,8 @@ jobs: # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39 # it is default that Travis PYPI provider has `skip_upload_docs: true` # set by default. + # Besides above, we don't do cleanup of `dist/*`, because it's being done + # by Travis CI PYPI deployment provider after upload, unconditionally. on: tags: true all_branches: true From f20b76b4ef54d146226836ac5e47501896c99fc8 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:13:20 +0300 Subject: [PATCH 111/167] Use sequence for before_cache step in YAML --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea8104b7d2c..f6a20331370 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,4 +100,5 @@ jobs: cache: pip -before_cache: rm -f $HOME/.cache/pip/log/debug.log +before_cache: +- rm -f $HOME/.cache/pip/log/debug.log From 8521a26234b30527c90522236766fd6fb9df4894 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:14:42 +0300 Subject: [PATCH 112/167] Add _reset_steps mixin to Travis config --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6a20331370..cb971c9b24e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,16 @@ _helpers: - &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python +- &_reset_steps + env: [] + before_install: skip + install: skip + script: skip + after_success: skip - &_doc_base stage: doc <<: *_mainstream_python_base + <<: *_reset_steps install: - *upgrade_python_toolset - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt @@ -61,6 +68,7 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + <<: *_reset_steps dist: trusty group: edge services: @@ -73,10 +81,8 @@ jobs: echo Not building wheels exit 0 fi - install: skip script: - ./tools/run_docker.sh "aiohttp" - after_success: skip deploy: provider: pypi # `skip_cleanup: true` is required to preserve binary wheels, built From c6cfbd5c6f73a12521434a42443e1678e7de9d9b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:15:16 +0300 Subject: [PATCH 113/167] Move default python versions matrix to root level --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb971c9b24e..fdb5c62f812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,14 @@ sudo: false language: python +python: +# python3.4.2 has bug in http.cookies module, aiohttp provides fix for it +- 3.4.2 +- 3.4.3 +- 3.5.2 +- 3.5 +- &mainstream_python 3.6 + install: - &upgrade_python_toolset pip install --upgrade pip setuptools wheel - pip install -U -r requirements/ci.txt @@ -19,7 +27,6 @@ after_success: - codecov _helpers: -- &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python - &_reset_steps @@ -52,13 +59,6 @@ jobs: - python: nightly include: - # python3.4.2 has bug in http.cookies module, aiohttp provides fix - - python: 3.4.2 - - python: 3.4.3 - - python: 3.5.2 - # - 3.5.3 - - python: 3.5 - - <<: *_mainstream_python_base - python: 3.6-dev - python: nightly @@ -67,7 +67,7 @@ jobs: - make doc-spelling - stage: deploy (PYPI upload itself runs only for tagged commits) - python: *mainstream_python + <<: *_mainstream_python_base <<: *_reset_steps dist: trusty group: edge From ccd7bfeb0a394a31a5a6644cc86af098bdd77b4a Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:21:05 +0300 Subject: [PATCH 114/167] Add myself to contributors as advised in checklist --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index d56ed9d096d..e49959e702a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -157,6 +157,7 @@ Stephen Granade Steven Seguin Sunghyun Hwang Sviatoslav Bulbakha +Sviatoslav Sydorenko Taha Jahangir Taras Voinarovskyi Terence Honles From 9aed572460344a16f28f45b6f72e3923ed65c572 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:16:35 +0200 Subject: [PATCH 115/167] Pin setuptools-git to latest version 1.2 (#2100) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 17c739b9bdf..8690e70ae01 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,4 +1,4 @@ -setuptools-git +setuptools-git==1.2 -r doc.txt pip==9.0.1 From a1b047e3edc9ab5bcbd50a7a1b9705098b3f3fcf Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:16:51 +0200 Subject: [PATCH 116/167] Pin codecov to latest version 2.0.9 (#2101) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 8690e70ae01..719a38d7a4a 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -22,5 +22,5 @@ yarl==0.12.0 # Using PEP 508 env markers to control dependency on runtimes: aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on windows -codecov; platform_system!="Windows" # We only use it in Travis CI +codecov==2.0.9; platform_system!="Windows" # We only use it in Travis CI uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 From eaa7bc78e5041367073b86227f467cfaf966157e Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:18:05 +0200 Subject: [PATCH 117/167] Pin uvloop to latest version 0.8.0 (#2102) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 719a38d7a4a..e8c6d52a54c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -23,4 +23,4 @@ yarl==0.12.0 # Using PEP 508 env markers to control dependency on runtimes: aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on windows codecov==2.0.9; platform_system!="Windows" # We only use it in Travis CI -uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 +uvloop==0.8.0; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 \ No newline at end of file From c9e4a19f987245ba1080ee7ca0676e73a27e0f3b Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:18:20 +0200 Subject: [PATCH 118/167] Pin sphinxcontrib-spelling to latest version 2.3.0 (#2103) --- requirements/doc-spelling.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt index 934d9426f0a..2226b066de4 100644 --- a/requirements/doc-spelling.txt +++ b/requirements/doc-spelling.txt @@ -1 +1 @@ -sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI +sphinxcontrib-spelling==2.3.0; platform_system!="Windows" # We only use it in Travis CI From 4fd6d997bb0d867df058cf63f8f15c9fc48e0fe5 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:18:32 +0200 Subject: [PATCH 119/167] Update pytest-mock from 1.6.0 to 1.6.1 (#2107) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index e8c6d52a54c..f3225c8a6e1 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -14,7 +14,7 @@ multidict==3.1.3 async-timeout==1.2.1 pytest==3.1.3 pytest-cov==2.5.1 -pytest-mock==1.6.0 +pytest-mock==1.6.1 gunicorn==19.7.1 twine==1.9.1 yarl==0.12.0 From af2f58df9a866cd2222379a6f3fcf5bdb3b953b7 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Mon, 17 Jul 2017 19:45:42 +0200 Subject: [PATCH 120/167] Update pytest-mock from 1.6.1 to 1.6.2 (#2108) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index f3225c8a6e1..5e240dc3a2c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -14,7 +14,7 @@ multidict==3.1.3 async-timeout==1.2.1 pytest==3.1.3 pytest-cov==2.5.1 -pytest-mock==1.6.1 +pytest-mock==1.6.2 gunicorn==19.7.1 twine==1.9.1 yarl==0.12.0 From 2d729172c7ba8294cc82bf9be8c6cc99468f0a2e Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Mon, 17 Jul 2017 12:25:04 -0700 Subject: [PATCH 121/167] update make clean --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index d8766fb088c..19021813b70 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,14 @@ clean: @rm -rf cover @make -C docs clean @python setup.py clean + @rm -f aiohttp/_frozenlist.html + @rm -f aiohttp/_frozenlist.c + @rm -f aiohttp/_frozenlist.*.so + @rm -f aiohttp/_frozenlist.*.pyd + @rm -f aiohttp/_http_parser.html + @rm -f aiohttp/_http_parser.c + @rm -f aiohttp/_http_parser.*.so + @rm -f aiohttp/_http_parser.*.pyd @rm -f aiohttp/_multidict.html @rm -f aiohttp/_multidict.c @rm -f aiohttp/_multidict.*.so From c7751e058472bc036160c52018ec56e797b707e7 Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Mon, 17 Jul 2017 13:02:50 -0700 Subject: [PATCH 122/167] calculate stringio size in bytes #2066 --- aiohttp/payload.py | 13 ++++++------- tests/test_payload.py | 9 +++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index b6334465ce7..e21b7bda72d 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -176,6 +176,12 @@ def __init__(self, value, *args, encoding=encoding, content_type=content_type, *args, **kwargs) +class StringIOPayload(StringPayload): + + def __init__(self, value, *args, **kwargs): + super().__init__(value.read(), *args, **kwargs) + + class IOBasePayload(Payload): def __init__(self, value, disposition='attachment', *args, **kwargs): @@ -236,13 +242,6 @@ def write(self, writer): self._value.close() -class StringIOPayload(TextIOPayload): - - @property - def size(self): - return len(self._value.getvalue()) - self._value.tell() - - class BytesIOPayload(IOBasePayload): @property diff --git a/tests/test_payload.py b/tests/test_payload.py index faf8d6de2b0..a786059b9fe 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -1,4 +1,5 @@ import asyncio +from io import StringIO import pytest @@ -56,3 +57,11 @@ def test_string_payload(): 'test', content_type='text/plain; charset=koi8-r') assert p.encoding == 'koi8-r' assert p.content_type == 'text/plain; charset=koi8-r' + + +def test_string_io_payload(): + s = StringIO('ű' * 5000) + p = payload.StringIOPayload(s) + assert p.encoding == 'utf-8' + assert p.content_type == 'text/plain; charset=utf-8' + assert p.size == 10000 From 4064cbd82ad0f2bc67b0b1187063d4ca8bcf2c1e Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Mon, 17 Jul 2017 13:30:56 -0700 Subject: [PATCH 123/167] handle CancelledError for response.write_eof() #2073 --- aiohttp/web_protocol.py | 3 +++ tests/test_web_server.py | 32 +++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 59c10116d2d..f4733600df6 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -471,6 +471,9 @@ def start(self, message, payload, handler): self.log_debug('Uncompleted request.') self.close() + except asyncio.CancelledError: + self.log_debug('Ignored premature client disconnection ') + break except RuntimeError as exc: if self.debug: self.log_exception( diff --git a/tests/test_web_server.py b/tests/test_web_server.py index 710d2b05162..a1579db570f 100644 --- a/tests/test_web_server.py +++ b/tests/test_web_server.py @@ -62,24 +62,38 @@ def handler(request): @asyncio.coroutine def test_raw_server_do_not_swallow_exceptions(raw_test_server, test_client): - exc = None @asyncio.coroutine def handler(request): - raise exc + raise asyncio.CancelledError() + + logger = mock.Mock() + server = yield from raw_test_server(handler, logger=logger) + cli = yield from test_client(server) + + with pytest.raises(client.ServerDisconnectedError): + yield from cli.get('/path/to') + + logger.debug.assert_called_with('Ignored premature client disconnection') + + +@asyncio.coroutine +def test_raw_server_cancelled_in_write_eof(raw_test_server, test_client): + + @asyncio.coroutine + def handler(request): + resp = web.Response(text=str(request.rel_url)) + resp.write_eof = mock.Mock(side_effect=asyncio.CancelledError("error")) + return resp logger = mock.Mock() server = yield from raw_test_server(handler, logger=logger) cli = yield from test_client(server) - for _exc, msg in ( - (asyncio.CancelledError("error"), - 'Ignored premature client disconnection'),): - exc = _exc - with pytest.raises(client.ServerDisconnectedError): - yield from cli.get('/path/to') + with pytest.raises(client.ServerDisconnectedError): + yield from cli.get('/path/to') - logger.debug.assert_called_with(msg) + logger.debug.assert_called_with('Ignored premature client disconnection ') @asyncio.coroutine From 7272c9f6a342aac4ab16fdf524c687d6907c25ff Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Fri, 21 Jul 2017 03:29:25 +0200 Subject: [PATCH 124/167] Update cython to 0.26 (#2115) * Update cython from 0.25.2 to 0.26 * Update cython from 0.25.2 to 0.26 --- requirements/ci.txt | 2 +- requirements/wheel.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 5e240dc3a2c..16a7f0095c1 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -6,7 +6,7 @@ flake8==3.3.0 pyflakes==1.5.0 coverage==4.4.1 cchardet==2.1.1 -cython==0.25.2 +cython==0.26 chardet==3.0.4 isort==4.2.15 tox==2.7.0 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index 5dba562b955..a9c4cc610d6 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ -cython==0.25.2 +cython==0.26 pytest==3.1.3 twine==1.9.1 From 240a0e2c82f8ef2efba4370f5c395b124ba04f0c Mon Sep 17 00:00:00 2001 From: jlacoline Date: Fri, 21 Jul 2017 03:31:07 +0200 Subject: [PATCH 125/167] Fix issue with IndexError being raised by the StreamReader.iter_chunks() generator (#2113) * stream.readchunk: update unit test to reflect expected behavior * stream.readchunk() bugfix: do not raise IndexError when buffer is empty * issue #2112: add news fragment to changes and update contributors * stream.readchunk: add a new unit test and change eof condition --- CONTRIBUTORS.txt | 1 + aiohttp/streams.py | 5 ++++- changes/2112.bugfix | 1 + tests/test_streams.py | 15 +++++++++++++-- 4 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 changes/2112.bugfix diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index e49959e702a..477554dd5d3 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -104,6 +104,7 @@ Kirill Klenov Kirill Malovitsa Kyrylo Perevozchikov Lars P. Søndergaard +Loïc Lajeanne Louis-Philippe Huberdeau Lu Gong Lubomir Gelo diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 9a0e01136a9..beb6a9d118c 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -326,7 +326,10 @@ def readchunk(self): if not self._buffer and not self._eof: yield from self._wait('readchunk') - return self._read_nowait_chunk(-1) + if self._buffer: + return self._read_nowait_chunk(-1) + else: + return b"" @asyncio.coroutine def readexactly(self, n): diff --git a/changes/2112.bugfix b/changes/2112.bugfix new file mode 100644 index 00000000000..1571e663cb9 --- /dev/null +++ b/changes/2112.bugfix @@ -0,0 +1 @@ +Fix issue with IndexError being raised by the StreamReader.iter_chunks() generator. diff --git a/tests/test_streams.py b/tests/test_streams.py index e7d545a8a08..31bea1f1a84 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -547,7 +547,6 @@ def test_read_nowait_waiter(self): self.assertRaises(RuntimeError, stream.read_nowait) def test_readchunk(self): - stream = self._make_one() def cb(): @@ -562,9 +561,21 @@ def cb(): data = self.loop.run_until_complete(stream.readchunk()) self.assertEqual(b'chunk2', data) - data = self.loop.run_until_complete(stream.read()) + data = self.loop.run_until_complete(stream.readchunk()) self.assertEqual(b'', data) + def test_readchunk_wait_eof(self): + stream = self._make_one() + + def cb(): + yield from asyncio.sleep(0.1, loop=self.loop) + stream.feed_eof() + + asyncio.Task(cb(), loop=self.loop) + data = self.loop.run_until_complete(stream.readchunk()) + self.assertEqual(b"", data) + self.assertTrue(stream.is_eof()) + def test___repr__(self): stream = self._make_one() self.assertEqual("", repr(stream)) From 631245887ab01d0211a827684bc555a5afc571a4 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Sun, 23 Jul 2017 16:08:15 +0100 Subject: [PATCH 126/167] add more details to ClientConnectorError --- aiohttp/client_exceptions.py | 19 +++++++++++++++++++ aiohttp/connector.py | 23 ++++++++++------------- tests/test_connector.py | 11 +++++++---- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 057d484904a..e3e6c3c9556 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -64,6 +64,25 @@ class ClientConnectorError(ClientOSError): Raised in :class:`aiohttp.connector.TCPConnector` if connection to proxy can not be established. """ + def __init__(self, connection_key, os_error): + self._conn_key = connection_key + super().__init__(os_error.errno, os_error.strerror) + + @property + def host(self): + return self._conn_key.host + + @property + def port(self): + return self._conn_key.port + + @property + def ssl(self): + return self._conn_key.ssl + + def __str__(self): + return ('Cannot connect to host {0.host}:{0.port} ssl:{0.ssl} [{1}]' + .format(self._conn_key, self.strerror)) class ClientProxyConnectionError(ClientConnectorError): diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 86c44c7358b..79974f9753a 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -4,7 +4,7 @@ import sys import traceback import warnings -from collections import defaultdict +from collections import defaultdict, namedtuple from hashlib import md5, sha1, sha256 from itertools import cycle, islice from time import monotonic @@ -128,6 +128,9 @@ def close(self): pass +ConnectionKey = namedtuple('ConnectionKey', ['host', 'port', 'ssl']) + + class BaseConnector(object): """Base connector class. @@ -300,11 +303,11 @@ def close(self): if self._loop.is_closed(): return noop() - # cacnel cleanup task + # cancel cleanup task if self._cleanup_handle: self._cleanup_handle.cancel() - # cacnel cleanup close task + # cancel cleanup close task if self._cleanup_closed_handle: self._cleanup_closed_handle.cancel() @@ -338,7 +341,7 @@ def closed(self): @asyncio.coroutine def connect(self, req): """Get from pool or create new connection.""" - key = (req.host, req.port, req.ssl) + key = ConnectionKey(req.host, req.port, req.ssl) if self._limit: # total calc available connections @@ -377,10 +380,7 @@ def connect(self, req): try: proto = yield from self._create_connection(req) except OSError as exc: - raise ClientConnectorError( - exc.errno, - 'Cannot connect to host {0[0]}:{0[1]} ssl:{0[2]} [{1}]' - .format(key, exc.strerror)) from exc + raise ClientConnectorError(key, exc) from exc finally: self._acquired.remove(placeholder) self._acquired_per_host[key].remove(placeholder) @@ -731,10 +731,7 @@ def _create_direct_connection(self, req): except OSError as e: exc = e else: - raise ClientConnectorError( - exc.errno, - 'Can not connect to %s:%s [%s]' % - (req.host, req.port, exc.strerror)) from exc + raise ClientConnectorError(req, exc) from exc @asyncio.coroutine def _create_proxy_connection(self, req): @@ -748,7 +745,7 @@ def _create_proxy_connection(self, req): transport, proto = yield from self._create_direct_connection( proxy_req) except OSError as exc: - raise ClientProxyConnectionError(*exc.args) from exc + raise ClientProxyConnectionError(proxy_req, exc) from exc auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) if auth is not None: diff --git a/tests/test_connector.py b/tests/test_connector.py index 6648bed0263..525bbc61776 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -499,19 +499,22 @@ def test_connect(loop): @asyncio.coroutine -def test_connect_oserr(loop): +def test_connect_connection_error(loop): conn = aiohttp.BaseConnector(loop=loop) conn._create_connection = mock.Mock() conn._create_connection.return_value = helpers.create_future(loop) err = OSError(1, 'permission error') conn._create_connection.return_value.set_exception(err) - with pytest.raises(aiohttp.ClientOSError) as ctx: + with pytest.raises(aiohttp.ClientConnectorError) as ctx: req = mock.Mock() yield from conn.connect(req) assert 1 == ctx.value.errno - assert ctx.value.strerror.startswith('Cannot connect to') - assert ctx.value.strerror.endswith('[permission error]') + assert str(ctx.value).startswith('Cannot connect to') + assert str(ctx.value).endswith('[permission error]') + assert ctx.value.host == req.host + assert ctx.value.port == req.port + assert ctx.value.ssl == req.ssl def test_ctor_cleanup(): From e2ce0bd36a9eebb4c6b874ffd1a649a07b2c5030 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Sun, 23 Jul 2017 16:21:11 +0100 Subject: [PATCH 127/167] add news fragment --- changes/2094.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/2094.misc diff --git a/changes/2094.misc b/changes/2094.misc new file mode 100644 index 00000000000..8832d0a8349 --- /dev/null +++ b/changes/2094.misc @@ -0,0 +1 @@ +add more details to ``ClientConnectorError`` exceptions about the original error. From 94de7f0c245ff440602f368026c74e62fd489c42 Mon Sep 17 00:00:00 2001 From: Arthur Darcet Date: Mon, 24 Jul 2017 18:21:35 +0200 Subject: [PATCH 128/167] log a warning when sending requests with a large body of bytes --- aiohttp/payload.py | 7 ++++++ tests/test_client_functional.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index e21b7bda72d..cab72a2eb84 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -3,6 +3,7 @@ import json import mimetypes import os +import warnings from abc import ABC, abstractmethod from multidict import CIMultiDict @@ -18,6 +19,7 @@ 'IOBasePayload', 'BytesIOPayload', 'BufferedReaderPayload', 'TextIOPayload', 'StringIOPayload', 'JsonPayload') +TOO_LARGE_BYTES_BODY = 2 ** 20 class LookupError(Exception): pass @@ -150,6 +152,11 @@ def __init__(self, value, *args, **kwargs): self._size = len(value) + if self._size > TOO_LARGE_BYTES_BODY: + warnings.warn("Sending a large body directly with raw bytes might" + " lock the event loop. You should probably pass an " + "io.BytesIO object instead", ResourceWarning) + @asyncio.coroutine def write(self, writer): yield from writer.write(self._value) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 48bcc9a2971..f5f47625004 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -1286,6 +1286,46 @@ def handler(request): resp.close() +@asyncio.coroutine +def test_POST_bytes(loop, test_client): + body = b'0' * 12345 + + @asyncio.coroutine + def handler(request): + data = yield from request.read() + assert body == data + return web.HTTPOk() + + app = web.Application() + app.router.add_post('/', handler) + client = yield from test_client(app) + + resp = yield from client.post('/', data=body) + assert 200 == resp.status + resp.close() + + +@asyncio.coroutine +def test_POST_bytes_too_large(loop, test_client): + body = b'0' * 2 ** 20 + 1 + + @asyncio.coroutine + def handler(request): + data = yield from request.read() + assert body == data + return web.HTTPOk() + + app = web.Application() + app.router.add_post('/', handler) + client = yield from test_client(app) + + with pytest.warns(ResourceWarning): + resp = yield from client.post('/', data=body) + + assert 200 == resp.status + resp.close() + + @asyncio.coroutine def test_POST_FILES_STR(loop, test_client, fname): @asyncio.coroutine From 4b3a60945cf5d3d3c01126826e49510145f6bdcd Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Mon, 24 Jul 2017 15:53:19 -0700 Subject: [PATCH 129/167] fix pep8 --- aiohttp/payload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index cab72a2eb84..52c529feb3a 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -21,6 +21,7 @@ TOO_LARGE_BYTES_BODY = 2 ** 20 + class LookupError(Exception): pass From 0de2c169f4d22a9589cb697e42aedafa93fd5648 Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Mon, 24 Jul 2017 16:15:37 -0700 Subject: [PATCH 130/167] fix tests for #2127 --- tests/test_client_functional.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index f5f47625004..127cabdf686 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -1307,11 +1307,11 @@ def handler(request): @asyncio.coroutine def test_POST_bytes_too_large(loop, test_client): - body = b'0' * 2 ** 20 + 1 + body = b'0' * (2 ** 20 + 1) @asyncio.coroutine def handler(request): - data = yield from request.read() + data = yield from request.content.read() assert body == data return web.HTTPOk() From b54f2e9af571573b6e63bf26e627d353dc423d07 Mon Sep 17 00:00:00 2001 From: Hynek Schlawack Date: Tue, 25 Jul 2017 10:58:25 +0200 Subject: [PATCH 131/167] Fix stacklevel in _CoroGuard's warning (#2125) Fixes #2106 --- CONTRIBUTORS.txt | 1 + aiohttp/helpers.py | 7 ++++++- changes/2106.bugfix | 1 + tests/test_client_session.py | 5 ++++- tests/test_helpers.py | 7 ++++++- 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 changes/2106.bugfix diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 477554dd5d3..e4ed80bcda4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -77,6 +77,7 @@ Gregory Haynes Günther Jena Hu Bo Hugo Herter +Hynek Schlawack Igor Davydenko Igor Pavlov Ingmar Steen diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 109343a5508..0bbffe44889 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -106,6 +106,11 @@ def __await__(self): class _CoroGuard(_BaseCoroMixin): + """Only to be used with func:`deprecated_noop`. + + Otherwise the stack information in the raised warning doesn't line up with + the user's code anymore. + """ __slots__ = ('_msg', '_awaited') def __init__(self, coro, msg): @@ -126,7 +131,7 @@ def __await__(self): def __del__(self): self._coro = None if not self._awaited: - warnings.warn(self._msg, DeprecationWarning) + warnings.warn(self._msg, DeprecationWarning, stacklevel=2) coroutines = asyncio.coroutines diff --git a/changes/2106.bugfix b/changes/2106.bugfix new file mode 100644 index 00000000000..d5e7b196f7f --- /dev/null +++ b/changes/2106.bugfix @@ -0,0 +1 @@ +Warnings about unawaited coroutines now correctly point to the user's code. diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 3eb99691938..d8c10449c67 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -64,9 +64,12 @@ def test_close_coro(create_session, loop): def test_close_deprecated(create_session): session = create_session() - with pytest.warns(DeprecationWarning): + with pytest.warns(DeprecationWarning) as ctx: session.close() + # Assert the warning points at us and not at _CoroGuard. + assert ctx.list[0].filename == __file__ + def test_init_headers_simple_dict(create_session): session = create_session(headers={"h1": "header1", diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 9514211dfe2..28f1140d340 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -17,7 +17,12 @@ def test_warn(): with pytest.warns(DeprecationWarning) as ctx: helpers.deprecated_noop('Text') - assert str(ctx.list[0].message) == 'Text' + + w = ctx.list[0] + + assert str(w.message) == 'Text' + # Assert the warning points at us and not at _CoroGuard. + assert w.filename == __file__ @asyncio.coroutine From 2fb3f641faa8032d3a7a9bbe02ebc8af250a00b7 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 25 Jul 2017 11:22:16 +0200 Subject: [PATCH 132/167] Added get_client and get_server methods to AioHTTPTestCase (#2074) * Added get_client and get_server methods to AioHTTPTestCase This change is needed if we want a clear distinction in the unittest.TestCase between "app" (the application) and "server" the actual test server. This will also allow a developer to override these methods to use their own TestServer and TestClient. * Added setUpAsync and tearDownAsync methods to AioHTTPTestCase --- aiohttp/pytest_plugin.py | 28 +++++++++------- aiohttp/test_utils.py | 48 +++++++++++++++------------ changes/2032.feature | 1 + docs/testing.rst | 28 ++++++++++++++++ tests/test_py35/test_test_utils_35.py | 2 +- tests/test_test_utils.py | 25 ++++---------- 6 files changed, 79 insertions(+), 53 deletions(-) create mode 100644 changes/2032.feature diff --git a/aiohttp/pytest_plugin.py b/aiohttp/pytest_plugin.py index 7fb2b041d63..ff1e23575a2 100644 --- a/aiohttp/pytest_plugin.py +++ b/aiohttp/pytest_plugin.py @@ -1,4 +1,5 @@ import asyncio +import collections import contextlib import tempfile import warnings @@ -9,8 +10,8 @@ from aiohttp.web import Application from .test_utils import unused_port as _unused_port -from .test_utils import (RawTestServer, TestClient, TestServer, loop_context, - setup_test_loop, teardown_test_loop) +from .test_utils import (BaseTestServer, RawTestServer, TestClient, TestServer, + loop_context, setup_test_loop, teardown_test_loop) try: @@ -227,19 +228,22 @@ def test_client(loop): clients = [] @asyncio.coroutine - def go(__param, *args, **kwargs): - if isinstance(__param, Application): - assert not args, "args should be empty" - client = TestClient(__param, loop=loop, **kwargs) - elif isinstance(__param, TestServer): - assert not args, "args should be empty" - client = TestClient(__param, loop=loop, **kwargs) - elif isinstance(__param, RawTestServer): + def go(__param, *args, server_kwargs={}, **kwargs): + + if isinstance(__param, collections.Callable) and \ + not isinstance(__param, (Application, BaseTestServer)): + __param = __param(loop, *args, **kwargs) + kwargs = {} + else: assert not args, "args should be empty" + + if isinstance(__param, Application): + server = TestServer(__param, loop=loop, **server_kwargs) + client = TestClient(server, loop=loop, **kwargs) + elif isinstance(__param, BaseTestServer): client = TestClient(__param, loop=loop, **kwargs) else: - __param = __param(loop, *args, **kwargs) - client = TestClient(__param, loop=loop) + raise ValueError("Unknown argument type: %r" % type(__param)) yield from client.start_server() clients.append(client) diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 5f8cce3cefe..4d9e7737681 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -20,7 +20,7 @@ from .helpers import PY_35, noop, sentinel from .http import HttpVersion, RawRequestMessage from .signals import Signal -from .web import Application, Request, Server, UrlMappingMatchInfo +from .web import Request, Server, UrlMappingMatchInfo def run_briefly(loop): @@ -185,23 +185,11 @@ class TestClient: """ - def __init__(self, app_or_server, *, scheme=sentinel, host=sentinel, - cookie_jar=None, server_kwargs=None, loop=None, **kwargs): - if isinstance(app_or_server, BaseTestServer): - if scheme is not sentinel or host is not sentinel: - raise ValueError("scheme and host are mutable exclusive " - "with TestServer parameter") - self._server = app_or_server - elif isinstance(app_or_server, Application): - scheme = "http" if scheme is sentinel else scheme - host = '127.0.0.1' if host is sentinel else host - server_kwargs = server_kwargs or {} - self._server = TestServer( - app_or_server, - scheme=scheme, host=host, **server_kwargs) - else: - raise TypeError("app_or_server should be either web.Application " - "or TestServer instance") + def __init__(self, server, *, cookie_jar=None, loop=None, **kwargs): + if not isinstance(server, BaseTestServer): + raise TypeError("server must be web.Application TestServer " + "instance, found type: %r" % type(server)) + self._server = server self._loop = loop if cookie_jar is None: cookie_jar = aiohttp.CookieJar(unsafe=True, loop=loop) @@ -390,18 +378,36 @@ def setUp(self): self.loop = setup_test_loop() self.app = self.loop.run_until_complete(self.get_application()) - self.client = self.loop.run_until_complete(self._get_client(self.app)) + self.server = self.loop.run_until_complete(self.get_server(self.app)) + self.client = self.loop.run_until_complete( + self.get_client(self.server)) self.loop.run_until_complete(self.client.start_server()) + self.loop.run_until_complete(self.setUpAsync()) + + @asyncio.coroutine + def setUpAsync(self): + pass + def tearDown(self): + self.loop.run_until_complete(self.tearDownAsync()) self.loop.run_until_complete(self.client.close()) teardown_test_loop(self.loop) @asyncio.coroutine - def _get_client(self, app): + def tearDownAsync(self): + pass + + @asyncio.coroutine + def get_server(self, app): + """Return a TestServer instance.""" + return TestServer(app, loop=self.loop) + + @asyncio.coroutine + def get_client(self, server): """Return a TestClient instance.""" - return TestClient(app, loop=self.loop) + return TestClient(server, loop=self.loop) def unittest_run_loop(func, *args, **kwargs): diff --git a/changes/2032.feature b/changes/2032.feature new file mode 100644 index 00000000000..9be68abe502 --- /dev/null +++ b/changes/2032.feature @@ -0,0 +1 @@ +Added `get_client`, `get_server`, `setUpAsync` and `tearDownAsync` methods to AioHTTPTestCase diff --git a/docs/testing.rst b/docs/testing.rst index 221e65cf228..7c98ca0b303 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -243,6 +243,10 @@ functionality, the AioHTTPTestCase is provided:: an aiohttp test client, :class:`TestClient` instance. + .. attribute:: server + + an aiohttp test server, :class:`TestServer` instance. + .. attribute:: loop The event loop in which the application and server are running. @@ -252,6 +256,20 @@ functionality, the AioHTTPTestCase is provided:: The application returned by :meth:`get_app` (:class:`aiohttp.web.Application` instance). + .. comethod:: get_client() + + This async method can be overridden to return the :class:`TestClient` + object used in the test. + + :return: :class:`TestClient` instance. + + .. comethod:: get_server() + + This async method can be overridden to return the :class:`TestServer` + object used in the test. + + :return: :class:`TestServer` instance. + .. comethod:: get_application() This async method should be overridden @@ -260,6 +278,16 @@ functionality, the AioHTTPTestCase is provided:: :return: :class:`aiohttp.web.Application` instance. + .. comethod:: setUpAsync() + + This async method do nothing by default and can be overriden to execute + asynchronous code during the setUp stage of the TestCase. + + .. comethod:: tearDownAsync() + + This async method do nothing by default and can be overriden to execute + asynchronous code during the tearDown stage of the TestCase. + .. method:: setUp() Standard test initialization method. diff --git a/tests/test_py35/test_test_utils_35.py b/tests/test_py35/test_test_utils_35.py index 73cd98248bc..a16ac61084c 100644 --- a/tests/test_py35/test_test_utils_35.py +++ b/tests/test_py35/test_test_utils_35.py @@ -27,7 +27,7 @@ async def test_server_context_manager(app, loop): "head", "get", "post", "options", "post", "put", "patch", "delete" ]) async def test_client_context_manager_response(method, app, loop): - async with _TestClient(app, loop=loop) as client: + async with _TestClient(_TestServer(app), loop=loop) as client: async with getattr(client, method)('/') as resp: assert resp.status == 200 if method != 'head': diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 916188f665a..cd2f1b8f164 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -50,7 +50,7 @@ def cookie_handler(request): def test_full_server_scenario(): with loop_context() as loop: app = _create_example_app() - with _TestClient(app, loop=loop) as client: + with _TestClient(_TestServer(app, loop=loop), loop=loop) as client: @asyncio.coroutine def test_get_route(): @@ -66,7 +66,7 @@ def test_get_route(): def test_server_with_create_test_teardown(): with loop_context() as loop: app = _create_example_app() - with _TestClient(app, loop=loop) as client: + with _TestClient(_TestServer(app, loop=loop), loop=loop) as client: @asyncio.coroutine def test_get_route(): @@ -85,7 +85,7 @@ def test_test_client_close_is_idempotent(): """ loop = setup_test_loop() app = _create_example_app() - client = _TestClient(app, loop=loop) + client = _TestClient(_TestServer(app, loop=loop), loop=loop) loop.run_until_complete(client.close()) loop.run_until_complete(client.close()) teardown_test_loop(loop) @@ -129,7 +129,7 @@ def app(): @pytest.yield_fixture def test_client(loop, app): - client = _TestClient(app, loop=loop) + client = _TestClient(_TestServer(app, loop=loop), loop=loop) loop.run_until_complete(client.start_server()) yield client loop.run_until_complete(client.close()) @@ -225,7 +225,8 @@ def test_make_mocked_request_transport(): def test_test_client_props(loop): app = _create_example_app() - client = _TestClient(app, loop=loop, host='localhost') + client = _TestClient(_TestServer(app, host='localhost', loop=loop), + loop=loop) assert client.host == 'localhost' assert client.port is None with client: @@ -248,20 +249,6 @@ def go(): loop.run_until_complete(go()) -def test_client_scheme_mutually_exclusive_with_server(): - app = _create_example_app() - server = _TestServer(app) - with pytest.raises(ValueError): - _TestClient(server, scheme='http') - - -def test_client_host_mutually_exclusive_with_server(): - app = _create_example_app() - server = _TestServer(app) - with pytest.raises(ValueError): - _TestClient(server, host='127.0.0.1') - - def test_client_unsupported_arg(): with pytest.raises(TypeError): _TestClient('string') From 4f033d3a61095818de8ee93ffb8a83ba29517bb6 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 25 Jul 2017 12:28:00 +0300 Subject: [PATCH 133/167] Fix docs --- docs/testing.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/testing.rst b/docs/testing.rst index 7c98ca0b303..97d85a176a6 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -247,6 +247,8 @@ functionality, the AioHTTPTestCase is provided:: an aiohttp test server, :class:`TestServer` instance. + .. versionadded:: 2.3.0 + .. attribute:: loop The event loop in which the application and server are running. @@ -263,6 +265,8 @@ functionality, the AioHTTPTestCase is provided:: :return: :class:`TestClient` instance. + .. versionadded:: 2.3.0 + .. comethod:: get_server() This async method can be overridden to return the :class:`TestServer` @@ -270,6 +274,8 @@ functionality, the AioHTTPTestCase is provided:: :return: :class:`TestServer` instance. + .. versionadded:: 2.3.0 + .. comethod:: get_application() This async method should be overridden @@ -280,13 +286,17 @@ functionality, the AioHTTPTestCase is provided:: .. comethod:: setUpAsync() - This async method do nothing by default and can be overriden to execute - asynchronous code during the setUp stage of the TestCase. + This async method do nothing by default and can be overridden to execute + asynchronous code during the ``setUp`` stage of the ``TestCase``. + + .. versionadded:: 2.3.0 .. comethod:: tearDownAsync() - This async method do nothing by default and can be overriden to execute - asynchronous code during the tearDown stage of the TestCase. + This async method do nothing by default and can be overridden to execute + asynchronous code during the ``tearDown`` stage of the ``TestCase``. + + .. versionadded:: 2.3.0 .. method:: setUp() From f59e5c48e845ded41114eebd59a8a44bf12a7e6e Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 25 Jul 2017 11:29:45 +0200 Subject: [PATCH 134/167] Force use of IPv4 during tests (#2105) --- changes/2104.bugfix | 2 ++ tests/test_run_app.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/2104.bugfix diff --git a/changes/2104.bugfix b/changes/2104.bugfix new file mode 100644 index 00000000000..896547273a3 --- /dev/null +++ b/changes/2104.bugfix @@ -0,0 +1,2 @@ +Force use of IPv4 during test, this will make tests run in a Docker container + diff --git a/tests/test_run_app.py b/tests/test_run_app.py index 4dcaf29d2d2..63fbe10d6bf 100644 --- a/tests/test_run_app.py +++ b/tests/test_run_app.py @@ -261,7 +261,7 @@ def test_run_app_nondefault_host_port(loop, unused_port, mocker): skip_if_no_dict(loop) port = unused_port() - host = 'localhost' + host = '127.0.0.1' mocker.spy(loop, 'create_server') From 8e76a4a89b04abf7d05da2d360867d6a7d531788 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 25 Jul 2017 11:31:00 +0200 Subject: [PATCH 135/167] Add automatically a SafeChildWatcher to the test loop (#2075) The fact that on Unix system we don't create automatically a child watcher makes all call to asyncio.create_subprocess_* to fail. --- aiohttp/test_utils.py | 6 ++++++ changes/2058.feature | 1 + tests/test_loop.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 changes/2058.feature create mode 100644 tests/test_loop.py diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 4d9e7737681..ac004cafc8d 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -5,6 +5,7 @@ import functools import gc import socket +import sys import unittest from abc import ABC, abstractmethod from contextlib import contextmanager @@ -446,6 +447,11 @@ def setup_test_loop(loop_factory=asyncio.new_event_loop): """ loop = loop_factory() asyncio.set_event_loop(None) + if sys.platform != "win32": + policy = asyncio.get_event_loop_policy() + watcher = asyncio.SafeChildWatcher() + watcher.attach_loop(loop) + policy.set_child_watcher(watcher) return loop diff --git a/changes/2058.feature b/changes/2058.feature new file mode 100644 index 00000000000..d6d5084c4fe --- /dev/null +++ b/changes/2058.feature @@ -0,0 +1 @@ +Add automatically a SafeChildWatcher to the test loop diff --git a/tests/test_loop.py b/tests/test_loop.py new file mode 100644 index 00000000000..d04dd56d9fc --- /dev/null +++ b/tests/test_loop.py @@ -0,0 +1,16 @@ +import asyncio +import platform +import threading + +import pytest + + +@pytest.mark.skipif(platform.system() == "Windows", + reason="the test is not valid for Windows") +@asyncio.coroutine +def test_subprocess_co(loop): + assert isinstance(threading.current_thread(), threading._MainThread) + proc = yield from asyncio.create_subprocess_shell( + "exit 0", loop=loop, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + yield from proc.wait() From 0d9abc9bf6391d92b44ece62a4ea509e997e450f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 25 Jul 2017 13:29:34 +0300 Subject: [PATCH 136/167] Fix #2072 #2067: Document connector_owner parameter --- changes/2072.doc | 1 + docs/client_reference.rst | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changes/2072.doc diff --git a/changes/2072.doc b/changes/2072.doc new file mode 100644 index 00000000000..964d094b929 --- /dev/null +++ b/changes/2072.doc @@ -0,0 +1 @@ +Document `connector_owner` parameter. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index c7925537d62..889d6cb41ef 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -43,11 +43,12 @@ The client session supports the context manager protocol for self closing. .. class:: ClientSession(*, connector=None, loop=None, cookies=None, \ headers=None, skip_auto_headers=None, \ - auth=None, json_serialize=func:`json.dumps`, \ + auth=None, json_serialize=json.dumps, \ version=aiohttp.HttpVersion11, \ cookie_jar=None, read_timeout=None, \ conn_timeout=None, \ - raise_for_status=False) + raise_for_status=False, \ + connector_owner=True) The class for creating client sessions and making requests. @@ -127,10 +128,15 @@ The client session supports the context manager protocol for self closing. :param float conn_timeout: timeout for connection establishing (optional). Values ``0`` or ``None`` mean no timeout. - .. versionchanged:: 1.0 + :param bool connector_owner: - ``.cookies`` attribute was dropped. Use :attr:`cookie_jar` - instead. + Close connector instance on session closing. + + Passing ``connector_owner=False`` to constructor allows to share + connection pool between sessions without sharing session state: + cookies etc. + + .. versionadded:: 2.1 .. attribute:: closed From 153197ff7e60bfe8125f8075220c4f8841c4b5df Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 25 Jul 2017 14:26:38 +0300 Subject: [PATCH 137/167] Fix #1993: Deprecate registering synchronous web handlers --- aiohttp/web_urldispatcher.py | 3 +++ changes/1993.feature | 1 + tests/test_urldispatch.py | 8 ++++++++ 3 files changed, 12 insertions(+) create mode 100644 changes/1993.feature diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 910f346e2c8..8f4f103d976 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -107,6 +107,9 @@ def __init__(self, method, handler, *, issubclass(handler, AbstractView)): pass else: + warnings.warn("Bare functions are deprecated, " + "use async ones", DeprecationWarning) + @wraps(handler) @asyncio.coroutine def handler_wrapper(*args, **kwargs): diff --git a/changes/1993.feature b/changes/1993.feature new file mode 100644 index 00000000000..e2eb6147fdf --- /dev/null +++ b/changes/1993.feature @@ -0,0 +1 @@ +Deprecate registering synchronous web handlers diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index c5b6b2f7934..8f5eae5af43 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -1049,3 +1049,11 @@ def test_convert_empty_path_to_slash_on_freezing(router): assert resource.get_info() == {'path': ''} router.freeze() assert resource.get_info() == {'path': '/'} + + +def test_deprecate_non_coroutine(router): + def handler(request): + pass + + with pytest.warns(DeprecationWarning): + router.add_route('GET', '/handler', handler) From 91dc5b7e3f757f062dd84ed3593971a16afb5efd Mon Sep 17 00:00:00 2001 From: Arthur Darcet Date: Wed, 26 Jul 2017 11:47:40 +0200 Subject: [PATCH 138/167] small optimization of BytesIOPayload.size (#2129) --- aiohttp/payload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index 52c529feb3a..a8b5c6f381e 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -254,7 +254,10 @@ class BytesIOPayload(IOBasePayload): @property def size(self): - return len(self._value.getbuffer()) - self._value.tell() + p = self._value.tell() + l = self._value.seek(0, os.SEEK_END) + self._value.seek(p) + return l - p class BufferedReaderPayload(IOBasePayload): From 07a4ef341cae3b8f09e6daa735b789bbe03f4160 Mon Sep 17 00:00:00 2001 From: Alexey Stepanov Date: Thu, 27 Jul 2017 11:08:03 +0300 Subject: [PATCH 139/167] Support HTTP 308 Permanent redirect in client class. (#2134) Implements: #2114 --- aiohttp/client.py | 3 ++- changes/2114.bugfix | 1 + tests/test_client_functional.py | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 changes/2114.bugfix diff --git a/aiohttp/client.py b/aiohttp/client.py index 4f3dad59de8..7f667caf677 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -251,7 +251,8 @@ def _request(self, method, url, *, self._cookie_jar.update_cookies(resp.cookies, resp.url) # redirects - if resp.status in (301, 302, 303, 307) and allow_redirects: + if resp.status in ( + 301, 302, 303, 307, 308) and allow_redirects: redirects += 1 history.append(resp) if max_redirects and redirects >= max_redirects: diff --git a/changes/2114.bugfix b/changes/2114.bugfix new file mode 100644 index 00000000000..451299998fd --- /dev/null +++ b/changes/2114.bugfix @@ -0,0 +1 @@ +Support HTTP 308 Permanent redirect in client class. \ No newline at end of file diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 127cabdf686..9b2bc20f925 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -970,6 +970,30 @@ def redirect(request): resp.close() +@asyncio.coroutine +def test_HTTP_308_PERMANENT_REDIRECT_POST(loop, test_client): + @asyncio.coroutine + def handler(request): + return web.Response(text=request.method) + + @asyncio.coroutine + def redirect(request): + yield from request.read() + return web.HTTPPermanentRedirect(location='/') + + app = web.Application() + app.router.add_post('/', handler) + app.router.add_post('/redirect', redirect) + client = yield from test_client(app) + + resp = yield from client.post('/redirect', data={'some': 'data'}) + assert 200 == resp.status + assert 1 == len(resp.history) + txt = yield from resp.text() + assert txt == 'POST' + resp.close() + + @asyncio.coroutine def test_HTTP_302_max_redirects(loop, test_client): @asyncio.coroutine From 6f9ffa7dc67796beda515d5ff5a58f42141da35c Mon Sep 17 00:00:00 2001 From: Pieter van Beek Date: Thu, 27 Jul 2017 10:25:34 +0200 Subject: [PATCH 140/167] Fixed issue #2118 (#2119) --- CONTRIBUTORS.txt | 1 + aiohttp/web_urldispatcher.py | 76 ++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index e4ed80bcda4..f5dc08ffc33 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -138,6 +138,7 @@ Paulus Schoutsen Pavel Kamaev Pawel Miech Philipp A. +Pieter van Beek Rafael Viotti Raúl Cumplido Required Field diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 8f4f103d976..9fc9cc506f5 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -31,6 +31,7 @@ 'StaticResource', 'View') HTTP_METHOD_RE = re.compile(r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$") +ROUTE_RE = re.compile(r'(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})') PATH_SEP = re.escape('/') @@ -333,11 +334,43 @@ def __repr__(self): class DynamicResource(Resource): - def __init__(self, pattern, formatter, *, name=None): + DYN = re.compile(r'\{(?P[_a-zA-Z][_a-zA-Z0-9]*)\}') + DYN_WITH_RE = re.compile( + r'\{(?P[_a-zA-Z][_a-zA-Z0-9]*):(?P.+)\}') + GOOD = r'[^{}/]+' + + def __init__(self, path, *, name=None): super().__init__(name=name) - assert pattern.pattern.startswith(PATH_SEP) + pattern = '' + formatter = '' + for part in ROUTE_RE.split(path): + match = self.DYN.fullmatch(part) + if match: + pattern += '(?P<{}>{})'.format(match.group('var'), self.GOOD) + formatter += '{' + match.group('var') + '}' + continue + + match = self.DYN_WITH_RE.fullmatch(part) + if match: + pattern += '(?P<{var}>{re})'.format(**match.groupdict()) + formatter += '{' + match.group('var') + '}' + continue + + if '{' in part or '}' in part: + raise ValueError("Invalid path '{}'['{}']".format(path, part)) + + path = URL(part).raw_path + formatter += path + pattern += re.escape(path) + + try: + compiled = re.compile(pattern) + except re.error as exc: + raise ValueError( + "Bad pattern '{}': {}".format(pattern, exc)) from None + assert compiled.pattern.startswith(PATH_SEP) assert formatter.startswith('/') - self._pattern = pattern + self._pattern = compiled self._formatter = formatter def add_prefix(self, prefix): @@ -705,11 +738,6 @@ def __contains__(self, route): class UrlDispatcher(AbstractRouter, collections.abc.Mapping): - DYN = re.compile(r'\{(?P[_a-zA-Z][_a-zA-Z0-9]*)\}') - DYN_WITH_RE = re.compile( - r'\{(?P[_a-zA-Z][_a-zA-Z0-9]*):(?P.+)\}') - GOOD = r'[^{}/]+' - ROUTE_RE = re.compile(r'(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})') NAME_SPLIT_RE = re.compile(r'[.:-]') def __init__(self): @@ -784,40 +812,12 @@ def register_resource(self, resource): def add_resource(self, path, *, name=None): if path and not path.startswith('/'): raise ValueError("path should be started with / or be empty") - if not ('{' in path or '}' in path or self.ROUTE_RE.search(path)): + if not ('{' in path or '}' in path or ROUTE_RE.search(path)): url = URL(path) resource = PlainResource(url.raw_path, name=name) self.register_resource(resource) return resource - - pattern = '' - formatter = '' - for part in self.ROUTE_RE.split(path): - match = self.DYN.fullmatch(part) - if match: - pattern += '(?P<{}>{})'.format(match.group('var'), self.GOOD) - formatter += '{' + match.group('var') + '}' - continue - - match = self.DYN_WITH_RE.fullmatch(part) - if match: - pattern += '(?P<{var}>{re})'.format(**match.groupdict()) - formatter += '{' + match.group('var') + '}' - continue - - if '{' in part or '}' in part: - raise ValueError("Invalid path '{}'['{}']".format(path, part)) - - path = URL(part).raw_path - formatter += path - pattern += re.escape(path) - - try: - compiled = re.compile(pattern) - except re.error as exc: - raise ValueError( - "Bad pattern '{}': {}".format(pattern, exc)) from None - resource = DynamicResource(compiled, formatter, name=name) + resource = DynamicResource(path, name=name) self.register_resource(resource) return resource From 9964ddbc68f7024fbd3895fe89c4a12e34b39b8b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 27 Jul 2017 12:55:01 +0300 Subject: [PATCH 141/167] Fix #2024: provide BaseRequest.loop attribute (#2132) * Fix #2024: provide BaseRequest.loop attribute * Add missing changes --- aiohttp/test_utils.py | 10 ++++++---- aiohttp/web.py | 1 + aiohttp/web_protocol.py | 2 +- aiohttp/web_request.py | 7 +++++++ aiohttp/web_server.py | 2 +- aiohttp/web_ws.py | 2 +- changes/2024.feature | 1 + docs/testing.rst | 6 +++++- docs/web_reference.rst | 27 +++++++++++++++++++-------- tests/test_web_websocket.py | 3 ++- 10 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 changes/2024.feature diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index ac004cafc8d..78b859c07e7 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -502,7 +502,8 @@ def make_mocked_request(method, path, headers=None, *, payload=sentinel, sslcontext=None, secure_proxy_ssl_header=None, - client_max_size=1024**2): + client_max_size=1024**2, + loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or @@ -511,8 +512,9 @@ def make_mocked_request(method, path, headers=None, *, """ task = mock.Mock() - loop = mock.Mock() - loop.create_future.return_value = () + if loop is ...: + loop = mock.Mock() + loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True @@ -566,7 +568,7 @@ def timeout(*args, **kw): time_service.timeout.side_effect = timeout req = Request(message, payload, - protocol, payload_writer, time_service, task, + protocol, payload_writer, time_service, task, loop, secure_proxy_ssl_header=secure_proxy_ssl_header, client_max_size=client_max_size) diff --git a/aiohttp/web.py b/aiohttp/web.py index c4afe4620c8..abbc096cda0 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -278,6 +278,7 @@ def _make_request(self, message, payload, protocol, writer, task, _cls=web_request.Request): return _cls( message, payload, protocol, writer, protocol._time_service, task, + self._loop, secure_proxy_ssl_header=self._secure_proxy_ssl_header, client_max_size=self._client_max_size) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index f4733600df6..0550df968da 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -555,7 +555,7 @@ def handle_error(self, request, status=500, exc=None, message=None): def handle_parse_error(self, writer, status, exc=None, message=None): request = BaseRequest( ERROR, EMPTY_PAYLOAD, - self, writer, self._time_service, None) + self, writer, self._time_service, None, self._loop) resp = self.handle_error(request, status, exc, message) yield from resp.prepare(request) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index b497b5a7f08..bcf7acddb1d 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -64,6 +64,7 @@ class BaseRequest(collections.MutableMapping, HeadersMixin): hdrs.METH_TRACE, hdrs.METH_DELETE} def __init__(self, message, payload, protocol, writer, time_service, task, + loop, *, secure_proxy_ssl_header=None, client_max_size=1024**2): self._message = message self._protocol = protocol @@ -84,6 +85,7 @@ def __init__(self, message, payload, protocol, writer, time_service, task, self._cache = {} self._task = task self._client_max_size = client_max_size + self._loop = loop def clone(self, *, method=sentinel, rel_url=sentinel, headers=sentinel): @@ -120,6 +122,7 @@ def clone(self, *, method=sentinel, rel_url=sentinel, self._writer, self._time_service, self._task, + self._loop, secure_proxy_ssl_header=self._secure_proxy_ssl_header) @property @@ -146,6 +149,10 @@ def message(self): def rel_url(self): return self._rel_url + @property + def loop(self): + return self._loop + # MutableMapping API def __getitem__(self, key): diff --git a/aiohttp/web_server.py b/aiohttp/web_server.py index b765d27849f..9a749565e4d 100644 --- a/aiohttp/web_server.py +++ b/aiohttp/web_server.py @@ -36,7 +36,7 @@ def connection_lost(self, handler, exc=None): def _make_request(self, message, payload, protocol, writer, task): return BaseRequest( message, payload, protocol, writer, - protocol.time_service, task) + protocol.time_service, task, self._loop) @asyncio.coroutine def shutdown(self, timeout=None): diff --git a/aiohttp/web_ws.py b/aiohttp/web_ws.py index 3f808765535..b5f30a2083d 100644 --- a/aiohttp/web_ws.py +++ b/aiohttp/web_ws.py @@ -100,7 +100,7 @@ def prepare(self, request): return payload_writer def _pre_start(self, request): - self._loop = request.app.loop + self._loop = request.loop try: status, headers, _, writer, protocol = do_handshake( diff --git a/changes/2024.feature b/changes/2024.feature new file mode 100644 index 00000000000..f4a927fe6a4 --- /dev/null +++ b/changes/2024.feature @@ -0,0 +1 @@ +Provide `BaseRequest.loop` attribute diff --git a/docs/testing.rst b/docs/testing.rst index 97d85a176a6..bf378fc8c5a 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -374,7 +374,8 @@ conditions that hard to reproduce on real server:: transport=sentinel, \ payload=sentinel, \ sslcontext=None, \ - secure_proxy_ssl_header=None) + secure_proxy_ssl_header=None, + loop=...) Creates mocked web.Request testing purposes. @@ -417,6 +418,9 @@ conditions that hard to reproduce on real server:: combination that signifies a request is secure. :type secure_proxy_ssl_header: tuple + :param loop: An event loop instance, mocked loop by default. + :type secure_proxy_ssl_header: :class:`asyncio.AbstractEventLoop` + :return: :class:`aiohttp.web.Request` object. diff --git a/docs/web_reference.rst b/docs/web_reference.rst index b4725eb3b37..07738a2630b 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -194,6 +194,14 @@ and :ref:`aiohttp-web-signals` handlers. if peername is not None: host, port = peername + .. attribute:: loop + + An event loop instance used by HTTP request handling. + + Read-only :class:`asyncio.AbstractEventLoop` property. + + .. versionadded:: 2.3 + .. attribute:: cookies A multidict of all request's cookies. @@ -820,7 +828,8 @@ Response WebSocketResponse ^^^^^^^^^^^^^^^^^ -.. class:: WebSocketResponse(*, timeout=10.0, receive_timeout=None, autoclose=True, \ +.. class:: WebSocketResponse(*, timeout=10.0, receive_timeout=None, \ + autoclose=True, \ autoping=True, heartbeat=None, protocols=()) Class for handling server-side websockets, inherited from @@ -834,8 +843,8 @@ WebSocketResponse .. versionadded:: 1.3.0 To enable back-pressure from slow websocket clients treat methods - `ping()`, `pong()`, `send_str()`, `send_bytes()`, `send_json()` as coroutines. - By default write buffer size is set to 64k. + `ping()`, `pong()`, `send_str()`, `send_bytes()`, `send_json()` as + coroutines. By default write buffer size is set to 64k. :param bool autoping: Automatically send :const:`~aiohttp.WSMsgType.PONG` on @@ -850,12 +859,14 @@ WebSocketResponse .. versionadded:: 1.3.0 - :param float heartbeat: Send `ping` message every `heartbeat` seconds - and wait `pong` response, close connection if `pong` response - is not received. + :param float heartbeat: Send `ping` message every `heartbeat` + seconds and wait `pong` response, close + connection if `pong` response is not + received. - :param float receive_timeout: Timeout value for `receive` operations. - Default value is None (no timeout for receive operation) + :param float receive_timeout: Timeout value for `receive` + operations. Default value is None + (no timeout for receive operation) .. versionadded:: 0.19 diff --git a/tests/test_web_websocket.py b/tests/test_web_websocket.py index 30cb584c147..7e793fd0302 100644 --- a/tests/test_web_websocket.py +++ b/tests/test_web_websocket.py @@ -51,7 +51,8 @@ def maker(method, path, headers=None, protocols=False): return make_mocked_request( method, path, headers, - app=app, protocol=protocol, payload_writer=writer) + app=app, protocol=protocol, payload_writer=writer, + loop=app.loop) return maker From ecc8e634cc6613001cb832707a8160eb7e97d4aa Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 27 Jul 2017 16:23:27 +0200 Subject: [PATCH 142/167] =?UTF-8?q?Raise=20ContentTypeError=20by=20json()?= =?UTF-8?q?=20when=20headers=20does=20not=20=E2=80=A6=20(#2136)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Raise ClientResponseContentTypeError by json() when headers does not meet the spec. * Docs changed to reflect the exception raised by json() * Added #2136 as a new feature change * Renamed long ClientResonseContentTypeError to ContentTypeError * Update client_exceptions.py --- aiohttp/client_exceptions.py | 6 ++++++ aiohttp/client_reqrep.py | 4 ++-- changes/2136.feature | 2 ++ docs/client_reference.rst | 10 +++++++--- tests/test_client_response.py | 13 +++++++++++-- 5 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 changes/2136.feature diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index e3e6c3c9556..72b0fc6d5d6 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -13,6 +13,8 @@ 'ServerFingerprintMismatch', 'ClientResponseError', 'ClientPayloadError', + 'ContentTypeError', + 'ClientHttpProxyError', 'WSServerHandshakeError') @@ -37,6 +39,10 @@ def __init__(self, request_info, history, *, super().__init__("%s, message='%s'" % (code, message)) +class ContentTypeError(ClientResponseError): + """ContentType found is not valid.""" + + class WSServerHandshakeError(ClientResponseError): """websocket server handshake error.""" diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 3fa52de8d52..5eff7de626a 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -13,7 +13,7 @@ from . import hdrs, helpers, http, payload from .client_exceptions import (ClientConnectionError, ClientOSError, - ClientResponseError) + ClientResponseError, ContentTypeError) from .formdata import FormData from .helpers import PY_35, HeadersMixin, SimpleCookie, TimerNoop, noop from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11, PayloadWriter @@ -722,7 +722,7 @@ def json(self, *, encoding=None, loads=json.loads, if content_type: ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower() if content_type not in ctype: - raise ClientResponseError( + raise ContentTypeError( self.request_info, self.history, message=('Attempt to decode JSON with ' diff --git a/changes/2136.feature b/changes/2136.feature new file mode 100644 index 00000000000..1aca0e0e2cb --- /dev/null +++ b/changes/2136.feature @@ -0,0 +1,2 @@ +json() raises a ContentTypeError exception if the content-type +does not meet the requirements instead of raising a generic ClientResponseError. diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 889d6cb41ef..553bea43a40 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1050,6 +1050,9 @@ Response object Close underlying connection if data reading gets an error, release connection otherwise. + Raise an :exc:`aiohttp.ClientResponseError` if the data can't + be read. + :return bytes: read *BODY*. .. seealso:: :meth:`close`, :meth:`release`. @@ -1104,15 +1107,16 @@ Response object content_type='application/json') Read response's body as *JSON*, return :class:`dict` using - specified *encoding* and *loader*. + specified *encoding* and *loader*. If data is not still available + a ``read`` call will be done, If *encoding* is ``None`` content encoding is autocalculated using :term:`cchardet` or :term:`chardet` as fallback if *cchardet* is not available. if response's `content-type` does not match `content_type` parameter - :exc:`aiohttp.ClientResponseError` get raised. To disable content type - check pass ``None`` value. + :exc:`aiohttp.ContentTypeError` get raised. + To disable content type check pass ``None`` value. :param str encoding: text encoding used for *BODY* decoding, or ``None`` for encoding autodetection diff --git a/tests/test_client_response.py b/tests/test_client_response.py index 5188730ae25..82b30792cec 100644 --- a/tests/test_client_response.py +++ b/tests/test_client_response.py @@ -352,18 +352,27 @@ def custom(content): @asyncio.coroutine -def test_json_no_content(loop, session): +def test_json_invalid_content_type(loop, session): response = ClientResponse('get', URL('http://def-cl-resp.org')) response._post_init(loop, session) response.headers = { 'Content-Type': 'data/octet-stream'} response._content = b'' - with pytest.raises(aiohttp.ClientResponseError) as info: + with pytest.raises(aiohttp.ContentTypeError) as info: yield from response.json() assert info.value.request_info == response.request_info + +@asyncio.coroutine +def test_json_no_content(loop, session): + response = ClientResponse('get', URL('http://def-cl-resp.org')) + response._post_init(loop, session) + response.headers = { + 'Content-Type': 'data/octet-stream'} + response._content = b'' + res = yield from response.json(content_type=None) assert res is None From 28ec7dfa8141517aca78248c2654aa6ae45af2b9 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 27 Jul 2017 20:39:53 +0300 Subject: [PATCH 143/167] Make dist folder be created under right privileges (#2137) * Make dist folder be created under right privileges * Move dist dir creation to sh script --- tools/run_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 1ea8c3a373a..bbc505fa766 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -16,6 +16,9 @@ do docker_pull_pids[$arch]=$! done +echo Creating dist folder with privileges of host-machine user +mkdir dist # This is required to be created with host-machine user privileges + for arch in x86_64 i686 do echo From 2400a4c722e6c3920122e9d6e23bc5120ab0b933 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 29 Jul 2017 15:00:18 +0300 Subject: [PATCH 144/167] Increase timeout for failed test --- tests/test_client_functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 9b2bc20f925..0008375f5c4 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -658,7 +658,7 @@ def handler(request): app.router.add_route('GET', '/', handler) client = yield from test_client(app) - resp = yield from client.get('/', timeout=0.1) + resp = yield from client.get('/', timeout=1) yield from fut with pytest.raises(asyncio.TimeoutError): From df64f926db7b910d99bb7555758916c86cf78f30 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Sat, 29 Jul 2017 16:55:26 +0200 Subject: [PATCH 145/167] Update flake8 from 3.3.0 to 3.4.1 (#2140) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 16a7f0095c1..6f70761bc1c 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -2,7 +2,7 @@ setuptools-git==1.2 -r doc.txt pip==9.0.1 -flake8==3.3.0 +flake8==3.4.1 pyflakes==1.5.0 coverage==4.4.1 cchardet==2.1.1 From f8fd3529b1ef315c2eebaa68f0573a2e3e533fd4 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 29 Jul 2017 18:18:32 +0300 Subject: [PATCH 146/167] Fix name of deploy job on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fdb5c62f812..77c30c5bb52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: script: - make doc-spelling - - stage: deploy (PYPI upload itself runs only for tagged commits) + - stage: deploy to PyPI <<: *_mainstream_python_base <<: *_reset_steps dist: trusty From bb646d143cc7bd3b1a614d7c101957b56b11a67f Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Sat, 29 Jul 2017 08:21:50 -0700 Subject: [PATCH 147/167] option to disable automatic client response body decompression (#2110) * work on feature to disable automatic response body decompression * add a missing folder to clean * bugfix * rename * add unittests * flake fixes * another pep fix * fix unittest * use constants * fix spelling * Update client_reference.rst Pin version to 2.3 * fix unittests * fix test * pep --- Makefile | 1 + aiohttp/_http_parser.pyx | 13 ++++---- aiohttp/client.py | 6 ++-- aiohttp/client_proto.py | 6 ++-- aiohttp/client_reqrep.py | 12 +++++--- aiohttp/http_parser.py | 18 ++++++++---- changes/2110.feature | 1 + docs/client_reference.rst | 7 ++++- tests/test_test_utils.py | 62 +++++++++++++++++++++++++++++++++------ 9 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 changes/2110.feature diff --git a/Makefile b/Makefile index 19021813b70..832ee53c4a9 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ clean: @rm -f .develop @rm -f .flake @rm -f .install-deps + @rm -rf aiohttp.egg-info doc: @make -C docs html SPHINXOPTS="-W -E" diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 4af5d20d21f..34f6c8c87d6 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -57,7 +57,8 @@ cdef class HttpParser: object _payload bint _payload_error object _payload_exception - object _last_error + object _last_error + bint _auto_decompress Py_buffer py_buf @@ -80,7 +81,7 @@ cdef class HttpParser: object protocol, object loop, object timer=None, size_t max_line_size=8190, size_t max_headers=32768, size_t max_field_size=8190, payload_exception=None, - response_with_body=True): + response_with_body=True, auto_decompress=True): cparser.http_parser_init(self._cparser, mode) self._cparser.data = self self._cparser.content_length = 0 @@ -106,6 +107,7 @@ cdef class HttpParser: self._max_field_size = max_field_size self._response_with_body = response_with_body self._upgraded = False + self._auto_decompress = auto_decompress self._csettings.on_url = cb_on_url self._csettings.on_status = cb_on_status @@ -194,7 +196,7 @@ cdef class HttpParser: payload = EMPTY_PAYLOAD self._payload = payload - if encoding is not None: + if encoding is not None and self._auto_decompress: self._payload = DeflateBuffer(payload, encoding) if not self._response_with_body: @@ -301,10 +303,11 @@ cdef class HttpResponseParserC(HttpParser): def __init__(self, protocol, loop, timer=None, size_t max_line_size=8190, size_t max_headers=32768, size_t max_field_size=8190, payload_exception=None, - response_with_body=True, read_until_eof=False): + response_with_body=True, read_until_eof=False, + auto_decompress=True): self._init(cparser.HTTP_RESPONSE, protocol, loop, timer, max_line_size, max_headers, max_field_size, - payload_exception, response_with_body) + payload_exception, response_with_body, auto_decompress) cdef int cb_on_message_begin(cparser.http_parser* parser) except -1: diff --git a/aiohttp/client.py b/aiohttp/client.py index 7f667caf677..a2d55a80e46 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -54,7 +54,8 @@ def __init__(self, *, connector=None, loop=None, cookies=None, ws_response_class=ClientWebSocketResponse, version=http.HttpVersion11, cookie_jar=None, connector_owner=True, raise_for_status=False, - read_timeout=sentinel, conn_timeout=None): + read_timeout=sentinel, conn_timeout=None, + auto_decompress=True): implicit_loop = False if loop is None: @@ -102,6 +103,7 @@ def __init__(self, *, connector=None, loop=None, cookies=None, else DEFAULT_TIMEOUT) self._conn_timeout = conn_timeout self._raise_for_status = raise_for_status + self._auto_decompress = auto_decompress # Convert to list of tuples if headers: @@ -223,7 +225,7 @@ def _request(self, method, url, *, expect100=expect100, loop=self._loop, response_class=self._response_class, proxy=proxy, proxy_auth=proxy_auth, timer=timer, - session=self) + session=self, auto_decompress=self._auto_decompress) # connection timeout try: diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 1a0baa3fd8d..09f875807f1 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -128,14 +128,16 @@ def set_parser(self, parser, payload): def set_response_params(self, *, timer=None, skip_payload=False, skip_status_codes=(), - read_until_eof=False): + read_until_eof=False, + auto_decompress=True): self._skip_payload = skip_payload self._skip_status_codes = skip_status_codes self._read_until_eof = read_until_eof self._parser = HttpResponseParser( self, self._loop, timer=timer, payload_exception=ClientPayloadError, - read_until_eof=read_until_eof) + read_until_eof=read_until_eof, + auto_decompress=auto_decompress) if self._tail: data, self._tail = self._tail, b'' diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 5eff7de626a..a038e265931 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -66,7 +66,7 @@ def __init__(self, method, url, *, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, proxy_from_env=False, - timer=None, session=None): + timer=None, session=None, auto_decompress=True): if loop is None: loop = asyncio.get_event_loop() @@ -88,6 +88,7 @@ def __init__(self, method, url, *, self.length = None self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else TimerNoop() + self._auto_decompress = auto_decompress if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) @@ -406,7 +407,8 @@ def send(self, conn): self.response = self.response_class( self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer, - request_info=self.request_info + request_info=self.request_info, + auto_decompress=self._auto_decompress ) self.response._post_init(self.loop, self._session) @@ -450,7 +452,7 @@ class ClientResponse(HeadersMixin): def __init__(self, method, url, *, writer=None, continue100=None, timer=None, - request_info=None): + request_info=None, auto_decompress=True): assert isinstance(url, URL) self.method = method @@ -465,6 +467,7 @@ def __init__(self, method, url, *, self._history = () self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() + self._auto_decompress = auto_decompress @property def url(self): @@ -550,7 +553,8 @@ def start(self, connection, read_until_eof=False): timer=self._timer, skip_payload=self.method.lower() == 'head', skip_status_codes=(204, 304), - read_until_eof=read_until_eof) + read_until_eof=read_until_eof, + auto_decompress=self._auto_decompress) with self._timer: while True: diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index c2fb6819229..eb6f0fd141a 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -59,7 +59,8 @@ def __init__(self, protocol=None, loop=None, max_line_size=8190, max_headers=32768, max_field_size=8190, timer=None, code=None, method=None, readall=False, payload_exception=None, - response_with_body=True, read_until_eof=False): + response_with_body=True, read_until_eof=False, + auto_decompress=True): self.protocol = protocol self.loop = loop self.max_line_size = max_line_size @@ -78,6 +79,7 @@ def __init__(self, protocol=None, loop=None, self._upgraded = False self._payload = None self._payload_parser = None + self._auto_decompress = auto_decompress def feed_eof(self): if self._payload_parser is not None: @@ -162,7 +164,8 @@ def feed_data(self, data, chunked=msg.chunked, method=method, compression=msg.compression, code=self.code, readall=self.readall, - response_with_body=self.response_with_body) + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress) if not payload_parser.done: self._payload_parser = payload_parser elif method == METH_CONNECT: @@ -171,7 +174,8 @@ def feed_data(self, data, self._upgraded = True self._payload_parser = HttpPayloadParser( payload, method=msg.method, - compression=msg.compression, readall=True) + compression=msg.compression, readall=True, + auto_decompress=self._auto_decompress) else: if (getattr(msg, 'code', 100) >= 199 and length is None and self.read_until_eof): @@ -182,7 +186,8 @@ def feed_data(self, data, chunked=msg.chunked, method=method, compression=msg.compression, code=self.code, readall=True, - response_with_body=self.response_with_body) + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress) if not payload_parser.done: self._payload_parser = payload_parser else: @@ -432,7 +437,7 @@ class HttpPayloadParser: def __init__(self, payload, length=None, chunked=False, compression=None, code=None, method=None, - readall=False, response_with_body=True): + readall=False, response_with_body=True, auto_decompress=True): self.payload = payload self._length = 0 @@ -440,10 +445,11 @@ def __init__(self, payload, self._chunk = ChunkState.PARSE_CHUNKED_SIZE self._chunk_size = 0 self._chunk_tail = b'' + self._auto_decompress = auto_decompress self.done = False # payload decompression wrapper - if (response_with_body and compression): + if response_with_body and compression and self._auto_decompress: payload = DeflateBuffer(payload, compression) # payload parser diff --git a/changes/2110.feature b/changes/2110.feature new file mode 100644 index 00000000000..31323a4a7d7 --- /dev/null +++ b/changes/2110.feature @@ -0,0 +1 @@ +add ability to disable automatic response decompression \ No newline at end of file diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 553bea43a40..e6cb1313b3f 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -48,7 +48,8 @@ The client session supports the context manager protocol for self closing. cookie_jar=None, read_timeout=None, \ conn_timeout=None, \ raise_for_status=False, \ - connector_owner=True) + connector_owner=True, \ + auto_decompress=True) The class for creating client sessions and making requests. @@ -138,6 +139,10 @@ The client session supports the context manager protocol for self closing. .. versionadded:: 2.1 + :param bool auto_decompress: Automatically decompress response body + + .. versionadded:: 2.3 + .. attribute:: closed ``True`` if the session has been closed, ``False`` otherwise. diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index cd2f1b8f164..621970b4154 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -1,4 +1,5 @@ import asyncio +import gzip from unittest import mock import pytest @@ -14,11 +15,20 @@ teardown_test_loop, unittest_run_loop) -def _create_example_app(): +_hello_world_str = "Hello, world" +_hello_world_bytes = _hello_world_str.encode('utf-8') +_hello_world_gz = gzip.compress(_hello_world_bytes) + +def _create_example_app(): @asyncio.coroutine def hello(request): - return web.Response(body=b"Hello, world") + return web.Response(body=_hello_world_bytes) + + @asyncio.coroutine + def gzip_hello(request): + return web.Response(body=_hello_world_gz, + headers={'Content-Encoding': 'gzip'}) @asyncio.coroutine def websocket_handler(request): @@ -36,12 +46,13 @@ def websocket_handler(request): @asyncio.coroutine def cookie_handler(request): - resp = web.Response(body=b"Hello, world") + resp = web.Response(body=_hello_world_bytes) resp.set_cookie('cookie', 'val') return resp app = web.Application() app.router.add_route('*', '/', hello) + app.router.add_route('*', '/gzip_hello', gzip_hello) app.router.add_route('*', '/websocket', websocket_handler) app.router.add_route('*', '/cookie', cookie_handler) return app @@ -58,7 +69,40 @@ def test_get_route(): resp = yield from client.request("GET", "/") assert resp.status == 200 text = yield from resp.text() - assert "Hello, world" in text + assert _hello_world_str == text + + loop.run_until_complete(test_get_route()) + + +def test_auto_gzip_decompress(): + with loop_context() as loop: + app = _create_example_app() + with _TestClient(_TestServer(app, loop=loop), loop=loop) as client: + + @asyncio.coroutine + def test_get_route(): + nonlocal client + resp = yield from client.request("GET", "/gzip_hello") + assert resp.status == 200 + data = yield from resp.read() + assert data == _hello_world_bytes + + loop.run_until_complete(test_get_route()) + + +def test_noauto_gzip_decompress(): + with loop_context() as loop: + app = _create_example_app() + with _TestClient(_TestServer(app, loop=loop), loop=loop, + auto_decompress=False) as client: + + @asyncio.coroutine + def test_get_route(): + nonlocal client + resp = yield from client.request("GET", "/gzip_hello") + assert resp.status == 200 + data = yield from resp.read() + assert data == _hello_world_gz loop.run_until_complete(test_get_route()) @@ -73,7 +117,7 @@ def test_get_route(): resp = yield from client.request("GET", "/") assert resp.status == 200 text = yield from resp.text() - assert "Hello, world" in text + assert _hello_world_str == text loop.run_until_complete(test_get_route()) @@ -102,7 +146,7 @@ def test_example_with_loop(self): request = yield from self.client.request("GET", "/") assert request.status == 200 text = yield from request.text() - assert "Hello, world" in text + assert _hello_world_str == text def test_example(self): @asyncio.coroutine @@ -110,7 +154,7 @@ def test_get_route(): resp = yield from self.client.request("GET", "/") assert resp.status == 200 text = yield from resp.text() - assert "Hello, world" in text + assert _hello_world_str == text self.loop.run_until_complete(test_get_route()) @@ -141,7 +185,7 @@ def test_get_route(): resp = yield from test_client.request("GET", "/") assert resp.status == 200 text = yield from resp.text() - assert "Hello, world" in text + assert _hello_world_str == text loop.run_until_complete(test_get_route()) @@ -176,7 +220,7 @@ def test_test_client_methods(method, loop, test_client): resp = yield from getattr(test_client, method)("/") assert resp.status == 200 text = yield from resp.text() - assert "Hello, world" in text + assert _hello_world_str == text @asyncio.coroutine From 44edff8a3a930e4ecc35773a48c300c29b9919ae Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 29 Jul 2017 18:25:25 +0300 Subject: [PATCH 148/167] Access logger improvements (#2145) * Fix #2123: User request instead of message in access logger * Doc update * Optimize request.host property * Introduce BaseRequest.remote property * Fix spelling errors * Add tests --- aiohttp/hdrs.py | 1 + aiohttp/helpers.py | 75 +++++----- aiohttp/web_protocol.py | 9 +- aiohttp/web_request.py | 36 ++++- changes/2123.feature | 4 + changes/2123.removal | 4 + docs/logging.rst | 4 - docs/spelling_wordlist.txt | 1 + docs/web_reference.rst | 15 ++ tests/test_helpers.py | 51 +++---- tests/test_web_request.py | 279 ++++++++++++++++++++++--------------- 11 files changed, 274 insertions(+), 205 deletions(-) create mode 100644 changes/2123.feature create mode 100644 changes/2123.removal diff --git a/aiohttp/hdrs.py b/aiohttp/hdrs.py index 0cb964136e5..e14fcfa43df 100644 --- a/aiohttp/hdrs.py +++ b/aiohttp/hdrs.py @@ -91,5 +91,6 @@ WANT_DIGEST = istr('WANT-DIGEST') WARNING = istr('WARNING') WWW_AUTHENTICATE = istr('WWW-AUTHENTICATE') +X_FORWARDED_FOR = istr('X-FORWARDED-FOR') X_FORWARDED_HOST = istr('X-FORWARDED-HOST') X_FORWARDED_PROTO = istr('X-FORWARDED-PROTO') diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 0bbffe44889..5a752ec91f4 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -306,7 +306,7 @@ class AccessLogger: log = logging.getLogger("spam") log_format = "%a %{User-Agent}i" access_logger = AccessLogger(log, log_format) - access_logger.log(message, environ, response, transport, time) + access_logger.log(request, response, time) Format: %% The percent sign @@ -337,7 +337,6 @@ class AccessLogger: 'D': 'request_time_micro', 'i': 'request_header', 'o': 'response_header', - 'e': 'environ' } LOG_FORMAT = '%a %l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"' @@ -408,76 +407,65 @@ def compile_format(self, log_format): return log_format, methods @staticmethod - def _format_e(key, args): - return (args[1] or {}).get(key, '-') - - @staticmethod - def _format_i(key, args): - if not args[0]: + def _format_i(key, request, response, time): + if request is None: return '(no headers)' # suboptimal, make istr(key) once - return args[0].headers.get(key, '-') + return request.headers.get(key, '-') @staticmethod - def _format_o(key, args): + def _format_o(key, request, response, time): # suboptimal, make istr(key) once - return args[2].headers.get(key, '-') + return response.headers.get(key, '-') @staticmethod - def _format_a(args): - if args[3] is None: + def _format_a(request, response, time): + if request is None: return '-' - peername = args[3].get_extra_info('peername') - if isinstance(peername, (list, tuple)): - return peername[0] - else: - return peername + ip = request.remote + return ip if ip is not None else '-' @staticmethod - def _format_t(args): + def _format_t(request, response, time): return datetime.datetime.utcnow().strftime('[%d/%b/%Y:%H:%M:%S +0000]') @staticmethod - def _format_P(args): + def _format_P(request, response, time): return "<%s>" % os.getpid() @staticmethod - def _format_r(args): - msg = args[0] - if not msg: + def _format_r(request, response, time): + if request is None: return '-' - return '%s %s HTTP/%s.%s' % tuple((msg.method, - msg.path) + msg.version) - - @staticmethod - def _format_s(args): - return args[2].status + return '%s %s HTTP/%s.%s' % tuple((request.method, + request.path_qs) + request.version) @staticmethod - def _format_b(args): - return args[2].body_length + def _format_s(request, response, time): + return response.status @staticmethod - def _format_O(args): - return args[2].body_length + def _format_b(request, response, time): + return response.body_length @staticmethod - def _format_T(args): - return round(args[4]) + def _format_T(request, response, time): + return round(time) @staticmethod - def _format_Tf(args): - return '%06f' % args[4] + def _format_Tf(request, response, time): + return '%06f' % time @staticmethod - def _format_D(args): - return round(args[4] * 1000000) + def _format_D(request, response, time): + return round(time * 1000000) - def _format_line(self, args): - return ((key, method(args)) for key, method in self._methods) + def _format_line(self, request, response, time): + return ((key, method(request, response, time)) + for key, method in self._methods) - def log(self, message, environ, response, transport, time): + def log(self, request, response, time): """Log access. :param message: Request object. May be None. @@ -487,8 +475,7 @@ def log(self, message, environ, response, transport, time): :param float time: Time taken to serve the request. """ try: - fmt_info = self._format_line( - [message, environ, response, transport, time]) + fmt_info = self._format_line(request, response, time) values = list() extra = dict() diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 0550df968da..ed86059421a 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -349,10 +349,9 @@ def force_close(self, send_last_heartbeat=False): self.transport.close() self.transport = None - def log_access(self, message, environ, response, time): - if self.access_logger: - self.access_logger.log(message, environ, response, - self.transport, time) + def log_access(self, request, response, time): + if self.access_logger is not None: + self.access_logger.log(request, response, time) def log_debug(self, *args, **kw): if self.debug: @@ -444,7 +443,7 @@ def start(self, message, payload, handler): # log access if self.access_log: - self.log_access(message, None, resp, loop.time() - now) + self.log_access(request, resp, loop.time() - now) # check payload if not payload.is_eof(): diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index bcf7acddb1d..d59c3a61c5a 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -275,7 +275,7 @@ def version(self): @reify def host(self): - """ Hostname of the request. + """Hostname of the request. Hostname is resolved through the following headers, in this order: @@ -288,12 +288,38 @@ def host(self): host = next( (f['host'] for f in self.forwarded if 'host' in f), None ) - if not host and hdrs.X_FORWARDED_HOST in self._message.headers: - host = self._message.headers[hdrs.X_FORWARDED_HOST] - elif hdrs.HOST in self._message.headers: - host = self._message.headers[hdrs.HOST] + if host is None: + host = self._message.headers.get(hdrs.X_FORWARDED_HOST) + if host is None: + host = self._message.headers.get(hdrs.HOST) return host + @reify + def remote(self): + """Remote IP of client initiated HTTP request. + + The IP is resolved through the following headers, in this order: + + - Forwarded + - X-Forwarded-For + - peername of opened socket + """ + ip = next( + (f['for'] for f in self.forwarded if 'for' in f), None + ) + if ip is None: + ips = self._message.headers.get(hdrs.X_FORWARDED_FOR) + if ips is not None: + ip = ips.split(',')[0].strip() + if ip is None: + transport = self._transport + peername = transport.get_extra_info('peername') + if isinstance(peername, (list, tuple)): + ip = peername[0] + else: + ip = peername + return ip + @reify def url(self): return URL('{}://{}{}'.format(self._scheme, diff --git a/changes/2123.feature b/changes/2123.feature new file mode 100644 index 00000000000..f7bb5e15687 --- /dev/null +++ b/changes/2123.feature @@ -0,0 +1,4 @@ +Use request for getting access log information instead of message/transport pair. + +Add `RequestBase.remote` property for accessing to IP of client +initiated HTTP request. diff --git a/changes/2123.removal b/changes/2123.removal new file mode 100644 index 00000000000..a7903db23da --- /dev/null +++ b/changes/2123.removal @@ -0,0 +1,4 @@ +Drop %O format from logging, use %b instead. + +Drop %e format from logging, environment variables are not supported anymore. + diff --git a/docs/logging.rst b/docs/logging.rst index ed1f6819be5..7695676ebb1 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -71,8 +71,6 @@ request and response: +--------------+---------------------------------------------------------+ | ``%b`` | Size of response in bytes, excluding HTTP headers | +--------------+---------------------------------------------------------+ -| ``%O`` | Bytes sent, including headers | -+--------------+---------------------------------------------------------+ | ``%T`` | The time taken to serve the request, in seconds | +--------------+---------------------------------------------------------+ | ``%Tf`` | The time taken to serve the request, in seconds | @@ -84,8 +82,6 @@ request and response: +--------------+---------------------------------------------------------+ | ``%{FOO}o`` | ``response.headers['FOO']`` | +--------------+---------------------------------------------------------+ -| ``%{FOO}e`` | ``os.environ['FOO']`` | -+--------------+---------------------------------------------------------+ Default access log format is:: diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d2b376190cd..ba6298aeaad 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -156,6 +156,7 @@ Paolini param params pathlib +peername ping pipelining pluggable diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 07738a2630b..1a25f0b5190 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -126,6 +126,21 @@ and :ref:`aiohttp-web-signals` handlers. Returns :class:`str`, or ``None`` if no host name is found in the headers. + .. attribute:: remote + + Originating IP address of a client initiated HTTP request. + + The IP is resolved through the following headers, in this order: + + - *Forwarded* + - *X-Forwarded-For* + - peer name of opened socket + + Returns :class:`str`, or ``None`` if no remote IP information is + provided. + + .. versionadded:: 2.3 + .. attribute:: path_qs The URL including PATH_INFO and the query string. e.g., diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 28f1140d340..fbfd3a5643f 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -131,10 +131,10 @@ def test_basic_auth_decode_bad_base64(): def test_access_logger_format(): - log_format = '%T {%{SPAM}e} "%{ETag}o" %X {X} %%P %{FOO_TEST}e %{FOO1}e' + log_format = '%T "%{ETag}o" %X {X} %%P' mock_logger = mock.Mock() access_logger = helpers.AccessLogger(mock_logger, log_format) - expected = '%s {%s} "%s" %%X {X} %%%s %s %s' + expected = '%s "%s" %%X {X} %%%s' assert expected == access_logger._log_format @@ -147,12 +147,11 @@ def test_access_logger_atoms(mocker): log_format = '%a %t %P %l %u %r %s %b %T %Tf %D' mock_logger = mock.Mock() access_logger = helpers.AccessLogger(mock_logger, log_format) - message = mock.Mock(headers={}, method="GET", path="/path", version=(1, 1)) - environ = {} + request = mock.Mock(headers={}, method="GET", path_qs="/path", + version=(1, 1), + remote="127.0.0.2") response = mock.Mock(headers={}, body_length=42, status=200) - transport = mock.Mock() - transport.get_extra_info.return_value = ("127.0.0.2", 1234) - access_logger.log(message, environ, response, transport, 3.1415926) + access_logger.log(request, response, 3.1415926) assert not mock_logger.exception.called expected = ('127.0.0.2 [01/Jan/1843:00:00:00 +0000] <42> - - ' 'GET /path HTTP/1.1 200 42 3 3.141593 3141593') @@ -171,19 +170,16 @@ def test_access_logger_atoms(mocker): def test_access_logger_dicts(): - log_format = '%{User-Agent}i %{Content-Length}o %{SPAM}e %{None}i' + log_format = '%{User-Agent}i %{Content-Length}o %{None}i' mock_logger = mock.Mock() access_logger = helpers.AccessLogger(mock_logger, log_format) - message = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1)) - environ = {"SPAM": "EGGS"} + request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), + remote="127.0.0.2") response = mock.Mock(headers={"Content-Length": 123}) - transport = mock.Mock() - transport.get_extra_info.return_value = ("127.0.0.2", 1234) - access_logger.log(message, environ, response, transport, 0.0) + access_logger.log(request, response, 0.0) assert not mock_logger.error.called - expected = 'Mock/1.0 123 EGGS -' + expected = 'Mock/1.0 123 -' extra = { - 'environ': {'SPAM': 'EGGS'}, 'request_header': {'None': '-'}, 'response_header': {'Content-Length': 123} } @@ -195,46 +191,39 @@ def test_access_logger_unix_socket(): log_format = '|%a|' mock_logger = mock.Mock() access_logger = helpers.AccessLogger(mock_logger, log_format) - message = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1)) - environ = {} + request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), + remote="") response = mock.Mock() - transport = mock.Mock() - transport.get_extra_info.return_value = "" - access_logger.log(message, environ, response, transport, 0.0) + access_logger.log(request, response, 0.0) assert not mock_logger.error.called expected = '||' mock_logger.info.assert_called_with(expected, extra={'remote_address': ''}) -def test_logger_no_message_and_environ(): +def test_logger_no_message(): mock_logger = mock.Mock() - mock_transport = mock.Mock() - mock_transport.get_extra_info.return_value = ("127.0.0.3", 0) access_logger = helpers.AccessLogger(mock_logger, - "%r %{FOOBAR}e %{content-type}i") + "%r %{content-type}i") extra_dict = { - 'environ': {'FOOBAR': '-'}, 'first_request_line': '-', 'request_header': {'content-type': '(no headers)'} } - access_logger.log(None, None, None, mock_transport, 0.0) - mock_logger.info.assert_called_with("- - (no headers)", extra=extra_dict) + access_logger.log(None, None, 0.0) + mock_logger.info.assert_called_with("- (no headers)", extra=extra_dict) def test_logger_internal_error(): mock_logger = mock.Mock() - mock_transport = mock.Mock() - mock_transport.get_extra_info.return_value = ("127.0.0.3", 0) access_logger = helpers.AccessLogger(mock_logger, "%D") - access_logger.log(None, None, None, mock_transport, 'invalid') + access_logger.log(None, None, 'invalid') mock_logger.exception.assert_called_with("Error in logging") def test_logger_no_transport(): mock_logger = mock.Mock() access_logger = helpers.AccessLogger(mock_logger, "%a") - access_logger.log(None, None, None, None, 0) + access_logger.log(None, None, 0) mock_logger.info.assert_called_with("-", extra={'remote_address': '-'}) diff --git a/tests/test_web_request.py b/tests/test_web_request.py index 36c16823cd0..e143b4e151b 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -12,13 +12,8 @@ from aiohttp.web import HTTPRequestEntityTooLarge -@pytest.fixture -def make_request(): - return make_mocked_request - - -def test_ctor(make_request): - req = make_request('GET', '/path/to?a=1&b=2') +def test_ctor(): + req = make_mocked_request('GET', '/path/to?a=1&b=2') assert 'GET' == req.method assert HttpVersion(1, 1) == req.version @@ -42,8 +37,8 @@ def test_ctor(make_request): payload = mock.Mock() protocol = mock.Mock() app = mock.Mock() - req = make_request('GET', '/path/to?a=1&b=2', headers=headers, - protocol=protocol, payload=payload, app=app) + req = make_mocked_request('GET', '/path/to?a=1&b=2', headers=headers, + protocol=protocol, payload=payload, app=app) assert req.app is app assert req.content is payload assert req.protocol is protocol @@ -56,83 +51,84 @@ def test_ctor(make_request): assert req.GET is req.query -def test_doubleslashes(make_request): +def test_doubleslashes(): # NB: //foo/bar is an absolute URL with foo netloc and /bar path - req = make_request('GET', '/bar//foo/') + req = make_mocked_request('GET', '/bar//foo/') assert '/bar//foo/' == req.path -def test_content_type_not_specified(make_request): - req = make_request('Get', '/') +def test_content_type_not_specified(): + req = make_mocked_request('Get', '/') assert 'application/octet-stream' == req.content_type -def test_content_type_from_spec(make_request): - req = make_request('Get', '/', - CIMultiDict([('CONTENT-TYPE', 'application/json')])) +def test_content_type_from_spec(): + req = make_mocked_request('Get', '/', + CIMultiDict([('CONTENT-TYPE', + 'application/json')])) assert 'application/json' == req.content_type -def test_content_type_from_spec_with_charset(make_request): - req = make_request( +def test_content_type_from_spec_with_charset(): + req = make_mocked_request( 'Get', '/', CIMultiDict([('CONTENT-TYPE', 'text/html; charset=UTF-8')])) assert 'text/html' == req.content_type assert 'UTF-8' == req.charset -def test_calc_content_type_on_getting_charset(make_request): - req = make_request( +def test_calc_content_type_on_getting_charset(): + req = make_mocked_request( 'Get', '/', CIMultiDict([('CONTENT-TYPE', 'text/html; charset=UTF-8')])) assert 'UTF-8' == req.charset assert 'text/html' == req.content_type -def test_urlencoded_querystring(make_request): - req = make_request('GET', - '/yandsearch?text=%D1%82%D0%B5%D0%BA%D1%81%D1%82') +def test_urlencoded_querystring(): + req = make_mocked_request( + 'GET', '/yandsearch?text=%D1%82%D0%B5%D0%BA%D1%81%D1%82') assert {'text': 'текст'} == req.query -def test_non_ascii_path(make_request): - req = make_request('GET', '/путь') +def test_non_ascii_path(): + req = make_mocked_request('GET', '/путь') assert '/путь' == req.path -def test_non_ascii_raw_path(make_request): - req = make_request('GET', '/путь') +def test_non_ascii_raw_path(): + req = make_mocked_request('GET', '/путь') assert '/путь' == req.raw_path -def test_content_length(make_request): - req = make_request('Get', '/', - CIMultiDict([('CONTENT-LENGTH', '123')])) +def test_content_length(): + req = make_mocked_request('Get', '/', + CIMultiDict([('CONTENT-LENGTH', '123')])) assert 123 == req.content_length -def test_non_keepalive_on_http10(make_request): - req = make_request('GET', '/', version=HttpVersion(1, 0)) +def test_non_keepalive_on_http10(): + req = make_mocked_request('GET', '/', version=HttpVersion(1, 0)) assert not req.keep_alive -def test_non_keepalive_on_closing(make_request): - req = make_request('GET', '/', closing=True) +def test_non_keepalive_on_closing(): + req = make_mocked_request('GET', '/', closing=True) assert not req.keep_alive @asyncio.coroutine -def test_call_POST_on_GET_request(make_request): - req = make_request('GET', '/') +def test_call_POST_on_GET_request(): + req = make_mocked_request('GET', '/') ret = yield from req.post() assert CIMultiDict() == ret @asyncio.coroutine -def test_call_POST_on_weird_content_type(make_request): - req = make_request( +def test_call_POST_on_weird_content_type(): + req = make_mocked_request( 'POST', '/', headers=CIMultiDict({'CONTENT-TYPE': 'something/weird'})) @@ -141,16 +137,16 @@ def test_call_POST_on_weird_content_type(make_request): @asyncio.coroutine -def test_call_POST_twice(make_request): - req = make_request('GET', '/') +def test_call_POST_twice(): + req = make_mocked_request('GET', '/') ret1 = yield from req.post() ret2 = yield from req.post() assert ret1 is ret2 -def test_no_request_cookies(make_request): - req = make_request('GET', '/') +def test_no_request_cookies(): + req = make_mocked_request('GET', '/') assert req.cookies == {} @@ -158,17 +154,17 @@ def test_no_request_cookies(make_request): assert cookies is req.cookies -def test_request_cookie(make_request): +def test_request_cookie(): headers = CIMultiDict(COOKIE='cookie1=value1; cookie2=value2') - req = make_request('GET', '/', headers=headers) + req = make_mocked_request('GET', '/', headers=headers) assert req.cookies == {'cookie1': 'value1', 'cookie2': 'value2'} -def test_request_cookie__set_item(make_request): +def test_request_cookie__set_item(): headers = CIMultiDict(COOKIE='name=value') - req = make_request('GET', '/', headers=headers) + req = make_mocked_request('GET', '/', headers=headers) assert req.cookies == {'name': 'value'} @@ -176,105 +172,109 @@ def test_request_cookie__set_item(make_request): req.cookies['my'] = 'value' -def test_match_info(make_request): - req = make_request('GET', '/') +def test_match_info(): + req = make_mocked_request('GET', '/') assert req._match_info is req.match_info -def test_request_is_mutable_mapping(make_request): - req = make_request('GET', '/') +def test_request_is_mutable_mapping(): + req = make_mocked_request('GET', '/') assert isinstance(req, MutableMapping) req['key'] = 'value' assert 'value' == req['key'] -def test_request_delitem(make_request): - req = make_request('GET', '/') +def test_request_delitem(): + req = make_mocked_request('GET', '/') req['key'] = 'value' assert 'value' == req['key'] del req['key'] assert 'key' not in req -def test_request_len(make_request): - req = make_request('GET', '/') +def test_request_len(): + req = make_mocked_request('GET', '/') assert len(req) == 0 req['key'] = 'value' assert len(req) == 1 -def test_request_iter(make_request): - req = make_request('GET', '/') +def test_request_iter(): + req = make_mocked_request('GET', '/') req['key'] = 'value' req['key2'] = 'value2' assert set(req) == {'key', 'key2'} -def test___repr__(make_request): - req = make_request('GET', '/path/to') +def test___repr__(): + req = make_mocked_request('GET', '/path/to') assert "" == repr(req) -def test___repr___non_ascii_path(make_request): - req = make_request('GET', '/path/\U0001f415\U0001f308') +def test___repr___non_ascii_path(): + req = make_mocked_request('GET', '/path/\U0001f415\U0001f308') assert "" == repr(req) -def test_http_scheme(make_request): - req = make_request('GET', '/') +def test_http_scheme(): + req = make_mocked_request('GET', '/') assert "http" == req.scheme assert req.secure is False -def test_https_scheme_by_ssl_transport(make_request): - req = make_request('GET', '/', sslcontext=True) +def test_https_scheme_by_ssl_transport(): + req = make_mocked_request('GET', '/', sslcontext=True) assert "https" == req.scheme assert req.secure is True -def test_https_scheme_by_secure_proxy_ssl_header(make_request): - req = make_request('GET', '/', - secure_proxy_ssl_header=('X-HEADER', '1'), - headers=CIMultiDict({'X-HEADER': '1'})) +def test_https_scheme_by_secure_proxy_ssl_header(): + req = make_mocked_request('GET', '/', + secure_proxy_ssl_header=('X-HEADER', '1'), + headers=CIMultiDict({'X-HEADER': '1'})) assert "https" == req.scheme assert req.secure is True -def test_https_scheme_by_secure_proxy_ssl_header_false_test(make_request): - req = make_request('GET', '/', - secure_proxy_ssl_header=('X-HEADER', '1'), - headers=CIMultiDict({'X-HEADER': '0'})) +def test_https_scheme_by_secure_proxy_ssl_header_false_test(): + req = make_mocked_request('GET', '/', + secure_proxy_ssl_header=('X-HEADER', '1'), + headers=CIMultiDict({'X-HEADER': '0'})) assert "http" == req.scheme assert req.secure is False -def test_single_forwarded_header(make_request): +def test_single_forwarded_header(): header = 'by=identifier;for=identifier;host=identifier;proto=identifier' - req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) assert req.forwarded[0]['by'] == 'identifier' assert req.forwarded[0]['for'] == 'identifier' assert req.forwarded[0]['host'] == 'identifier' assert req.forwarded[0]['proto'] == 'identifier' -def test_single_forwarded_header_camelcase(make_request): +def test_single_forwarded_header_camelcase(): header = 'bY=identifier;fOr=identifier;HOst=identifier;pRoTO=identifier' - req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) assert req.forwarded[0]['by'] == 'identifier' assert req.forwarded[0]['for'] == 'identifier' assert req.forwarded[0]['host'] == 'identifier' assert req.forwarded[0]['proto'] == 'identifier' -def test_single_forwarded_header_single_param(make_request): +def test_single_forwarded_header_single_param(): header = 'BY=identifier' - req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) assert req.forwarded[0]['by'] == 'identifier' -def test_single_forwarded_header_multiple_param(make_request): +def test_single_forwarded_header_multiple_param(): header = 'By=identifier1,BY=identifier2, By=identifier3 , BY=identifier4' - req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) assert len(req.forwarded) == 4 assert req.forwarded[0]['by'] == 'identifier1' assert req.forwarded[1]['by'] == 'identifier2' @@ -282,18 +282,19 @@ def test_single_forwarded_header_multiple_param(make_request): assert req.forwarded[3]['by'] == 'identifier4' -def test_single_forwarded_header_quoted_escaped(make_request): +def test_single_forwarded_header_quoted_escaped(): header = 'BY=identifier;pROTO="\lala lan\d\~ 123\!&"' - req = make_request('GET', '/', headers=CIMultiDict({'Forwarded': header})) + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) assert req.forwarded[0]['by'] == 'identifier' assert req.forwarded[0]['proto'] == 'lala land~ 123!&' -def test_multiple_forwarded_headers(make_request): +def test_multiple_forwarded_headers(): headers = CIMultiDict() headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') headers.add('Forwarded', 'By=identifier4;fOr=identifier5') - req = make_request('GET', '/', headers=headers) + req = make_mocked_request('GET', '/', headers=headers) assert len(req.forwarded) == 3 assert req.forwarded[0]['by'] == 'identifier1' assert req.forwarded[0]['for'] == 'identifier2' @@ -302,75 +303,79 @@ def test_multiple_forwarded_headers(make_request): assert req.forwarded[2]['for'] == 'identifier5' -def test_https_scheme_by_forwarded_header(make_request): - req = make_request('GET', '/', - headers=CIMultiDict( - {'Forwarded': 'by=;for=;host=;proto=https'})) +def test_https_scheme_by_forwarded_header(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict( + {'Forwarded': 'by=;for=;host=;proto=https'})) assert "https" == req.scheme assert req.secure is True -def test_https_scheme_by_malformed_forwarded_header(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'Forwarded': 'malformed value'})) +def test_https_scheme_by_malformed_forwarded_header(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': + 'malformed value'})) assert "http" == req.scheme assert req.secure is False -def test_https_scheme_by_x_forwarded_proto_header(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'X-Forwarded-Proto': 'https'})) +def test_https_scheme_by_x_forwarded_proto_header(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'X-Forwarded-Proto': + 'https'})) assert "https" == req.scheme assert req.secure is True -def test_https_scheme_by_x_forwarded_proto_header_no_tls(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'X-Forwarded-Proto': 'http'})) +def test_https_scheme_by_x_forwarded_proto_header_no_tls(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'X-Forwarded-Proto': + 'http'})) assert "http" == req.scheme assert req.secure is False -def test_host_by_forwarded_header(make_request): +def test_host_by_forwarded_header(): headers = CIMultiDict() headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') headers.add('Forwarded', 'by=;for=;host=example.com') - req = make_request('GET', '/', headers=headers) + req = make_mocked_request('GET', '/', headers=headers) assert req.host == 'example.com' -def test_host_by_forwarded_header_malformed(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'Forwarded': 'malformed value'})) +def test_host_by_forwarded_header_malformed(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': + 'malformed value'})) assert req.host is None -def test_host_by_x_forwarded_host_header(make_request): - req = make_request('GET', '/', - headers=CIMultiDict( - {'X-Forwarded-Host': 'example.com'})) +def test_host_by_x_forwarded_host_header(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict( + {'X-Forwarded-Host': 'example.com'})) assert req.host == 'example.com' -def test_host_by_host_header(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'Host': 'example.com'})) +def test_host_by_host_header(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Host': 'example.com'})) assert req.host == 'example.com' -def test_raw_headers(make_request): - req = make_request('GET', '/', - headers=CIMultiDict({'X-HEADER': 'aaa'})) +def test_raw_headers(): + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'X-HEADER': 'aaa'})) assert req.raw_headers == ((b'X-HEADER', b'aaa'),) -def test_rel_url(make_request): - req = make_request('GET', '/path') +def test_rel_url(): + req = make_mocked_request('GET', '/path') assert URL('/path') == req.rel_url -def test_url_url(make_request): - req = make_request('GET', '/path', headers={'HOST': 'example.com'}) +def test_url_url(): + req = make_mocked_request('GET', '/path', headers={'HOST': 'example.com'}) assert URL('http://example.com/path') == req.url @@ -487,3 +492,45 @@ def test_make_too_big_request_limit_None(loop): client_max_size=max_size) txt = yield from req.read() assert len(txt) == 1024**2 + 1 + + +def test_remote_peername_tcp(): + transp = mock.Mock() + transp.get_extra_info.return_value = ('10.10.10.10', 1234) + req = make_mocked_request('GET', '/', transport=transp) + assert req.remote == '10.10.10.10' + + +def test_remote_peername_unix(): + transp = mock.Mock() + transp.get_extra_info.return_value = '/path/to/sock' + req = make_mocked_request('GET', '/', transport=transp) + assert req.remote == '/path/to/sock' + + +def test_remote_peername_x_forwarded(): + transp = mock.Mock() + transp.get_extra_info.return_value = ('10.10.10.10', 1234) + req = make_mocked_request( + 'GET', '/', transport=transp, + headers={'X-Forwarded-For': '11.11.11.11, 12.12.12.12'}) + assert req.remote == '11.11.11.11' + + +def test_remote_peername_forwarded(): + transp = mock.Mock() + transp.get_extra_info.return_value = ('10.10.10.10', 1234) + req = make_mocked_request( + 'GET', '/', transport=transp, + headers={'Forwarded': 'for=11.11.11.11, for=12.12.12.12'}) + assert req.remote == '11.11.11.11' + + +def test_remote_peername_forwarded_overrides_x_forwarded(): + transp = mock.Mock() + transp.get_extra_info.return_value = ('10.10.10.10', 1234) + req = make_mocked_request( + 'GET', '/', transport=transp, + headers={'Forwarded': 'for=11.11.11.11, for=12.12.12.12', + 'X-Forwarded-For': '13.13.13.13'}) + assert req.remote == '11.11.11.11' From e7b62d1edc1b26605192bd2f36b12c7329a15de0 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Mon, 31 Jul 2017 07:04:47 +0200 Subject: [PATCH 149/167] Throttle simultaneous DNS requests #1924 (#2111) * Throttle simultaneous DNS requests Added a new option at TCPConnector level called `throttle_dns` that allows to the connector throttle simulatnewous DNS requests for a specific domain. This functionality its only compatible when the DNS cache is enabled and it is dissabled by default. Therefore, many requests can end up making the same DNS resolution. * Explicit loop for sleep calls * Fixed typos in documentation * Added DNS request throttling in CHANGES.rst * Dogpile invalid word by doc-spelling * Get full coverage for locks * Throttle DNS always enabled Will help the user to throttle DNS requests to the same domain. By design it will happen always when many URLs behind the same domain are scheduled at the same time, perhaps via *gather* The Event class is no longer a derivated class of the asyncio.locks.Event, and makes a full copy of this one to avoid future issues. * Remove implicit loop tests for locks.Event * Use create_future helper * Updated spelling wordlist * Use a simple Event wrapper instead of a full Event class implementation * Pass the loop explicitly * Cancel pending throttled DNS requests if connector is close * Removed not used attribute * Pass loop explicit * Throtlle DNS feature adapted to CHANGES protocol * Update 2111.feature * Changed class name --- CHANGES.rst | 1 + aiohttp/connector.py | 47 ++++++++++++++++----- aiohttp/locks.py | 42 +++++++++++++++++++ changes/2111.feature | 3 ++ docs/spelling_wordlist.txt | 12 ++++++ tests/test_connector.py | 83 +++++++++++++++++++++++++++++++++++--- tests/test_locks.py | 62 ++++++++++++++++++++++++++++ 7 files changed, 234 insertions(+), 16 deletions(-) create mode 100644 aiohttp/locks.py create mode 100644 changes/2111.feature create mode 100644 tests/test_locks.py diff --git a/CHANGES.rst b/CHANGES.rst index 49887998871..d72af723ae8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,7 @@ Changes .. towncrier release notes start + 2.2.3 (2017-07-04) ================== diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 79974f9753a..dfc2fcd09a9 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -17,6 +17,7 @@ from .client_proto import ResponseHandler from .client_reqrep import ClientRequest from .helpers import SimpleCookie, is_ip_address, noop, sentinel +from .locks import EventResultOrError from .resolver import DefaultResolver @@ -597,10 +598,18 @@ def __init__(self, *, verify_ssl=True, fingerprint=None, self._use_dns_cache = use_dns_cache self._cached_hosts = _DNSCacheTable(ttl=ttl_dns_cache) + self._throttle_dns_events = {} self._ssl_context = ssl_context self._family = family self._local_addr = local_addr + def close(self): + """Close all ongoing DNS calls.""" + for ev in self._throttle_dns_events.values(): + ev.cancel() + + super().close() + @property def verify_ssl(self): """Do check for ssl certifications?""" @@ -660,20 +669,38 @@ def _resolve_host(self, host, port): return [{'hostname': host, 'host': host, 'port': port, 'family': self._family, 'proto': 0, 'flags': 0}] - if self._use_dns_cache: - key = (host, port) + if not self._use_dns_cache: + return (yield from self._resolver.resolve( + host, port, family=self._family)) - if key not in self._cached_hosts or\ - self._cached_hosts.expired(key): - addrs = yield from \ - self._resolver.resolve(host, port, family=self._family) - self._cached_hosts.add(key, addrs) + key = (host, port) + if (key in self._cached_hosts) and\ + (not self._cached_hosts.expired(key)): return self._cached_hosts.next_addrs(key) + + if key in self._throttle_dns_events: + yield from self._throttle_dns_events[key].wait() else: - res = yield from self._resolver.resolve( - host, port, family=self._family) - return res + self._throttle_dns_events[key] = \ + EventResultOrError(self._loop) + try: + addrs = yield from \ + asyncio.shield(self._resolver.resolve(host, + port, + family=self._family), + loop=self._loop) + self._cached_hosts.add(key, addrs) + self._throttle_dns_events[key].set() + except Exception as e: + # any DNS exception, independently of the implementation + # is set for the waiters to raise the same exception. + self._throttle_dns_events[key].set(exc=e) + raise + finally: + self._throttle_dns_events.pop(key) + + return self._cached_hosts.next_addrs(key) @asyncio.coroutine def _create_connection(self, req): diff --git a/aiohttp/locks.py b/aiohttp/locks.py new file mode 100644 index 00000000000..96ac097e433 --- /dev/null +++ b/aiohttp/locks.py @@ -0,0 +1,42 @@ +import asyncio +import collections + +from .helpers import ensure_future + + +class EventResultOrError: + """ + This class wrappers the Event asyncio lock allowing either awake the + locked Tasks without any error or raising an exception. + + thanks to @vorpalsmith for the simple design. + """ + def __init__(self, loop): + self._loop = loop + self._exc = None + self._event = asyncio.Event(loop=loop) + self._waiters = collections.deque() + + def set(self, exc=None): + self._exc = exc + self._event.set() + + @asyncio.coroutine + def wait(self): + fut = ensure_future(self._event.wait(), loop=self._loop) + self._waiters.append(fut) + try: + val = yield from fut + finally: + self._waiters.remove(fut) + + if self._exc is not None: + raise self._exc + + return val + + def cancel(self): + """ Cancel all waiters """ + for fut in self._waiters: + if not fut.done(): + fut.cancel() diff --git a/changes/2111.feature b/changes/2111.feature new file mode 100644 index 00000000000..d11f0e74544 --- /dev/null +++ b/changes/2111.feature @@ -0,0 +1,3 @@ +Add support for throttling DNS request, avoiding the requests saturation +when there is a miss in the DNS cache and many requests getting into the +connector at the same time. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index ba6298aeaad..cfd034405f7 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -2,11 +2,14 @@ abc aiodns aioes aiohttp +aiohttp’s aiohttpdemo aiopg alives api +api’s app +app’s arg Arsenic async @@ -84,6 +87,7 @@ gethostbyname github google gunicorn +gunicorn’s Gunicorn gzipped hackish @@ -135,10 +139,12 @@ msg MsgType multidict multidicts +multidict’s Multidicts multipart Multipart Nagle +Nagle’s namedtuple nameservers namespace @@ -174,6 +180,7 @@ pyenv pyflakes pytest Pytest +quote’s readonly readpayload rebase @@ -186,11 +193,14 @@ refactoring regex regexps regexs +request’s +Request’s reloader renderer renderers repo repr +repr’s RequestContextManager requote resolvers @@ -245,6 +255,7 @@ url urldispatcher urlencoded urls +url’s utf utils uvloop @@ -253,6 +264,7 @@ waituntil webapp websocket websockets +websocket’s Websockets wildcard Workflow diff --git a/tests/test_connector.py b/tests/test_connector.py index 525bbc61776..680d8730b0c 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -361,13 +361,18 @@ def test_tcp_connector_resolve_host(loop): assert rec['host'] == '::1' -@asyncio.coroutine -def dns_response(): - return ["127.0.0.1"] +@pytest.fixture +def dns_response(loop): + @asyncio.coroutine + def coro(): + # simulates a network operation + yield from asyncio.sleep(0, loop=loop) + return ["127.0.0.1"] + return coro @asyncio.coroutine -def test_tcp_connector_dns_cache_not_expired(loop): +def test_tcp_connector_dns_cache_not_expired(loop, dns_response): with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: conn = aiohttp.TCPConnector( loop=loop, @@ -385,7 +390,7 @@ def test_tcp_connector_dns_cache_not_expired(loop): @asyncio.coroutine -def test_tcp_connector_dns_cache_forever(loop): +def test_tcp_connector_dns_cache_forever(loop, dns_response): with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: conn = aiohttp.TCPConnector( loop=loop, @@ -403,7 +408,7 @@ def test_tcp_connector_dns_cache_forever(loop): @asyncio.coroutine -def test_tcp_connector_use_dns_cache_disabled(loop): +def test_tcp_connector_use_dns_cache_disabled(loop, dns_response): with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: conn = aiohttp.TCPConnector(loop=loop, use_dns_cache=False) m_resolver().resolve.return_value = dns_response() @@ -415,6 +420,72 @@ def test_tcp_connector_use_dns_cache_disabled(loop): ]) +@asyncio.coroutine +def test_tcp_connector_dns_throttle_requests(loop, dns_response): + with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: + conn = aiohttp.TCPConnector( + loop=loop, + use_dns_cache=True, + ttl_dns_cache=10 + ) + m_resolver().resolve.return_value = dns_response() + helpers.ensure_future(conn._resolve_host('localhost', 8080), loop=loop) + helpers.ensure_future(conn._resolve_host('localhost', 8080), loop=loop) + yield from asyncio.sleep(0, loop=loop) + m_resolver().resolve.assert_called_once_with( + 'localhost', + 8080, + family=0 + ) + + +@asyncio.coroutine +def test_tcp_connector_dns_throttle_requests_exception_spread(loop): + with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: + conn = aiohttp.TCPConnector( + loop=loop, + use_dns_cache=True, + ttl_dns_cache=10 + ) + e = Exception() + m_resolver().resolve.side_effect = e + r1 = helpers.ensure_future( + conn._resolve_host('localhost', 8080), + loop=loop + ) + r2 = helpers.ensure_future( + conn._resolve_host('localhost', 8080), + loop=loop + ) + yield from asyncio.sleep(0, loop=loop) + assert r1.exception() == e + assert r2.exception() == e + + +@asyncio.coroutine +def test_tcp_connector_dns_throttle_requests_cancelled_when_close( + loop, + dns_response): + + with mock.patch('aiohttp.connector.DefaultResolver') as m_resolver: + conn = aiohttp.TCPConnector( + loop=loop, + use_dns_cache=True, + ttl_dns_cache=10 + ) + m_resolver().resolve.return_value = dns_response() + helpers.ensure_future( + conn._resolve_host('localhost', 8080), loop=loop) + f = helpers.ensure_future( + conn._resolve_host('localhost', 8080), loop=loop) + + yield from asyncio.sleep(0, loop=loop) + conn.close() + + with pytest.raises(asyncio.futures.CancelledError): + yield from f + + def test_get_pop_empty_conns(loop): # see issue #473 conn = aiohttp.BaseConnector(loop=loop) diff --git a/tests/test_locks.py b/tests/test_locks.py new file mode 100644 index 00000000000..764f8ab83a0 --- /dev/null +++ b/tests/test_locks.py @@ -0,0 +1,62 @@ +"""Tests of custom aiohttp locks implementations""" +import asyncio + +import pytest + +from aiohttp import helpers +from aiohttp.locks import EventResultOrError + + +class TestEventResultOrError: + + @asyncio.coroutine + def test_set_exception(self, loop): + ev = EventResultOrError(loop=loop) + + @asyncio.coroutine + def c(): + try: + yield from ev.wait() + except Exception as e: + return e + return 1 + + t = helpers.ensure_future(c(), loop=loop) + yield from asyncio.sleep(0, loop=loop) + e = Exception() + ev.set(exc=e) + assert (yield from t) == e + + @asyncio.coroutine + def test_set(self, loop): + ev = EventResultOrError(loop=loop) + + @asyncio.coroutine + def c(): + yield from ev.wait() + return 1 + + t = helpers.ensure_future(c(), loop=loop) + yield from asyncio.sleep(0, loop=loop) + ev.set() + assert (yield from t) == 1 + + @asyncio.coroutine + def test_cancel_waiters(self, loop): + ev = EventResultOrError(loop=loop) + + @asyncio.coroutine + def c(): + yield from ev.wait() + + t1 = helpers.ensure_future(c(), loop=loop) + t2 = helpers.ensure_future(c(), loop=loop) + yield from asyncio.sleep(0, loop=loop) + ev.cancel() + ev.set() + + with pytest.raises(asyncio.futures.CancelledError): + yield from t1 + + with pytest.raises(asyncio.futures.CancelledError): + yield from t2 From d522a28fa3bbc74ba624707a58a787419069f31e Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Mon, 31 Jul 2017 15:09:08 +0200 Subject: [PATCH 150/167] use osx postgres user (#2149) * use osx postgres user * Create 2149.misc --- changes/2149.misc | 1 + demos/polls/sql/install.sh | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 changes/2149.misc diff --git a/changes/2149.misc b/changes/2149.misc new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/changes/2149.misc @@ -0,0 +1 @@ + diff --git a/demos/polls/sql/install.sh b/demos/polls/sql/install.sh index b1207c4eb0c..8869f01a431 100755 --- a/demos/polls/sql/install.sh +++ b/demos/polls/sql/install.sh @@ -1,8 +1,15 @@ -sudo -u postgres psql -c "DROP DATABASE IF EXISTS aiohttpdemo_polls" -sudo -u postgres psql -c "DROP ROLE IF EXISTS aiohttpdemo_user" -sudo -u postgres psql -c "CREATE USER aiohttpdemo_user WITH PASSWORD 'aiohttpdemo_user';" -sudo -u postgres psql -c "CREATE DATABASE aiohttpdemo_polls ENCODING 'UTF8';" -sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE aiohttpdemo_polls TO aiohttpdemo_user;" +# determine os +unameOut="$(uname -s)" +case "${unameOut}" in + Darwin*) pg_cmd="psql -U postgres";; + *) pg_cmd="sudo -u postgres psql" +esac -cat sql/create_tables.sql | sudo -u postgres psql -d aiohttpdemo_polls -a -cat sql/sample_data.sql | sudo -u postgres psql -d aiohttpdemo_polls -a +${pg_cmd} -c "DROP DATABASE IF EXISTS aiohttpdemo_polls" +${pg_cmd} -c "DROP ROLE IF EXISTS aiohttpdemo_user" +${pg_cmd} -c "CREATE USER aiohttpdemo_user WITH PASSWORD 'aiohttpdemo_user';" +${pg_cmd} -c "CREATE DATABASE aiohttpdemo_polls ENCODING 'UTF8';" +${pg_cmd} -c "GRANT ALL PRIVILEGES ON DATABASE aiohttpdemo_polls TO aiohttpdemo_user;" + +cat sql/create_tables.sql | ${pg_cmd} -d aiohttpdemo_polls -a +cat sql/sample_data.sql | ${pg_cmd} -d aiohttpdemo_polls -a From e2836d1bc32df3e08b4b814e492f5f70a73ae5df Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Fri, 28 Jul 2017 16:32:59 +0200 Subject: [PATCH 151/167] Display a hint if you forget to define get_application() --- aiohttp/test_utils.py | 2 +- tests/conftest.py | 2 +- tests/test_test_utils.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 78b859c07e7..d5ea7a6a6e6 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -373,7 +373,7 @@ def get_app(self): Use .get_application() coroutine instead """ - pass # pragma: no cover + raise RuntimeError("Did you forget to define get_application()?") def setUp(self): self.loop = setup_test_loop() diff --git a/tests/conftest.py b/tests/conftest.py index 429fc743705..23fd1c3c2c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest -pytest_plugins = 'aiohttp.pytest_plugin' +pytest_plugins = ['aiohttp.pytest_plugin', 'pytester'] _LoggingWatcher = collections.namedtuple("_LoggingWatcher", diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 621970b4154..d8ea68cc07a 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -307,3 +307,17 @@ def test_server_make_url_yarl_compatibility(loop): make_url('http://foo.com') with pytest.raises(AssertionError): make_url(URL('http://foo.com')) + + +def test_testcase_no_app(testdir, loop): + testdir.makepyfile( + """ + from aiohttp.test_utils import AioHTTPTestCase + + + class InvalidTestCase(AioHTTPTestCase): + def test_noop(self): + pass + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines(["*RuntimeError*"]) From fa0ef995ad92317135f75607763bbbf8e68cab04 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Wed, 2 Aug 2017 07:58:11 +0200 Subject: [PATCH 152/167] Update pytest to 3.2.0 (#2153) * Update pytest from 3.1.3 to 3.2.0 * Update pytest from 3.1.3 to 3.2.0 --- requirements/ci.txt | 2 +- requirements/wheel.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 6f70761bc1c..da86e4636df 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -12,7 +12,7 @@ isort==4.2.15 tox==2.7.0 multidict==3.1.3 async-timeout==1.2.1 -pytest==3.1.3 +pytest==3.2.0 pytest-cov==2.5.1 pytest-mock==1.6.2 gunicorn==19.7.1 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index a9c4cc610d6..ce60d0dab98 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ cython==0.26 -pytest==3.1.3 +pytest==3.2.0 twine==1.9.1 From a1121368425037e703c8ba47bd43468db2c71aee Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 2 Aug 2017 11:49:16 +0300 Subject: [PATCH 153/167] Fix skipped tests --- tests/test_web_response.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 9df6670d555..72c0cc754a6 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -771,6 +771,7 @@ def test_keep_alive_http09(): assert not resp.keep_alive +@asyncio.coroutine def test_prepare_twice(): req = make_request('GET', '/') resp = StreamResponse() @@ -816,6 +817,7 @@ def test_get_nodelay_prepared(): assert not resp.tcp_nodelay +@asyncio.coroutine def test_set_nodelay_prepared(): resp = StreamResponse() writer = mock.Mock() @@ -849,6 +851,7 @@ def test_get_cork_prepared(): assert not resp.tcp_cork +@asyncio.coroutine def test_set_cork_prepared(): resp = StreamResponse() writer = mock.Mock() From f8a1536f21362b28c90d5c36bfcd26282ca13bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=BE=D1=80=D0=B5=D0=BD=D0=B1=D0=B5=D1=80=D0=B3=20?= =?UTF-8?q?=D0=9C=D0=B0=D1=80=D0=BA?= Date: Wed, 2 Aug 2017 20:35:38 +0500 Subject: [PATCH 154/167] Document order of calling signal handlers (#2155) (#2156) --- docs/web.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/web.rst b/docs/web.rst index 3e121c80cb1..0b6803302ca 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -980,6 +980,8 @@ This can be accomplished by subscribing to the Signal handlers should not return a value but may modify incoming mutable parameters. +Signal handlers will be run sequentially, in order they were added. If handler +is asynchronous, it will be awaited before calling next one. .. warning:: From edc4a0bc6a8a9304d765b1f833716574effcda48 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 2 Aug 2017 18:39:58 +0300 Subject: [PATCH 155/167] Move a chapter about migration to aiohttp 2 into essays --- docs/essays.rst | 1 + docs/index.rst | 1 - docs/{migration.rst => migration_to_2xx.rst} | 76 ++++++++++++-------- 3 files changed, 46 insertions(+), 32 deletions(-) rename docs/{migration.rst => migration_to_2xx.rst} (73%) diff --git a/docs/essays.rst b/docs/essays.rst index 7329ff6ba28..75bd3090d84 100644 --- a/docs/essays.rst +++ b/docs/essays.rst @@ -6,3 +6,4 @@ Essays new_router whats_new_1_1 + migration_to_2xx diff --git a/docs/index.rst b/docs/index.rst index 63d30c2022c..75baced29c5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -199,7 +199,6 @@ Contents .. toctree:: - migration client client_reference tutorial diff --git a/docs/migration.rst b/docs/migration_to_2xx.rst similarity index 73% rename from docs/migration.rst rename to docs/migration_to_2xx.rst index 6684c104301..a9883ba84e9 100644 --- a/docs/migration.rst +++ b/docs/migration_to_2xx.rst @@ -13,9 +13,10 @@ aiohttp does not support custom chunking sizes. It is up to the developer to decide how to chunk data streams. If chunking is enabled, aiohttp encodes the provided chunks in the "Transfer-encoding: chunked" format. -aiohttp does not enable chunked encoding automatically even if a *transfer-encoding* -header is supplied: *chunked* has to be set explicitly. If *chunked* is set, -then the *Transfer-encoding* and *content-length* headers are disallowed. +aiohttp does not enable chunked encoding automatically even if a +*transfer-encoding* header is supplied: *chunked* has to be set +explicitly. If *chunked* is set, then the *Transfer-encoding* and +*content-length* headers are disallowed. compression ^^^^^^^^^^^ @@ -29,21 +30,24 @@ Compression can not be combined with a *Content-Length* header. Client Connector ^^^^^^^^^^^^^^^^ -1. By default a connector object manages a total number of concurrent connections. - This limit was a per host rule in version 1.x. In 2.x, the `limit` parameter - defines how many concurrent connection connector can open and a new `limit_per_host` - parameter defines the limit per host. By default there is no per-host limit. -2. BaseConnector.close is now a normal function as opposed to coroutine in version 1.x +1. By default a connector object manages a total number of concurrent + connections. This limit was a per host rule in version 1.x. In + 2.x, the `limit` parameter defines how many concurrent connection + connector can open and a new `limit_per_host` parameter defines the + limit per host. By default there is no per-host limit. +2. BaseConnector.close is now a normal function as opposed to + coroutine in version 1.x 3. BaseConnector.conn_timeout was moved to ClientSession ClientResponse.release ^^^^^^^^^^^^^^^^^^^^^^ -Internal implementation was significantly redesigned. It is not required -to call `release` on the response object. When the client fully receives the payload, -the underlying connection automatically returns back to pool. If the payload is not -fully read, the connection is closed +Internal implementation was significantly redesigned. It is not +required to call `release` on the response object. When the client +fully receives the payload, the underlying connection automatically +returns back to pool. If the payload is not fully read, the connection +is closed Client exceptions @@ -54,25 +58,30 @@ exceptions that covers connection handling and server response misbehaviors. For developer specific mistakes, aiohttp uses python standard exceptions like ValueError or TypeError. -Reading a response content may raise a ClientPayloadError exception. This exception -indicates errors specific to the payload encoding. Such as invalid compressed data, -malformed chunked-encoded chunks or not enough data that satisfy the content-length header. +Reading a response content may raise a ClientPayloadError +exception. This exception indicates errors specific to the payload +encoding. Such as invalid compressed data, malformed chunked-encoded +chunks or not enough data that satisfy the content-length header. -All exceptions are moved from `aiohttp.errors` module to top level `aiohttp` module. +All exceptions are moved from `aiohttp.errors` module to top level +`aiohttp` module. New hierarchy of exceptions: * `ClientError` - Base class for all client specific exceptions - - `ClientResponseError` - exceptions that could happen after we get response from server + - `ClientResponseError` - exceptions that could happen after we get + response from server * `WSServerHandshakeError` - web socket server response error - `ClientHttpProxyError` - proxy response - - `ClientConnectionError` - exceptions related to low-level connection problems + - `ClientConnectionError` - exceptions related to low-level + connection problems - * `ClientOSError` - subset of connection errors that are initiated by an OSError exception + * `ClientOSError` - subset of connection errors that are initiated + by an OSError exception - `ClientConnectorError` - connector related exceptions @@ -86,24 +95,26 @@ New hierarchy of exceptions: * `ServerFingerprintMismatch` - server fingerprint mismatch - - `ClientPayloadError` - This exception can only be raised while reading the response - payload if one of these errors occurs: invalid compression, malformed chunked encoding or - not enough data that satisfy content-length header. + - `ClientPayloadError` - This exception can only be raised while + reading the response payload if one of these errors occurs: + invalid compression, malformed chunked encoding or not enough data + that satisfy content-length header. Client payload (form-data) ^^^^^^^^^^^^^^^^^^^^^^^^^^ -To unify form-data/payload handling a new `Payload` system was introduced. It handles -customized handling of existing types and provide implementation for user-defined types. +To unify form-data/payload handling a new `Payload` system was +introduced. It handles customized handling of existing types and +provide implementation for user-defined types. 1. FormData.__call__ does not take an encoding arg anymore and its return value changes from an iterator or bytes to a Payload instance. aiohttp provides payload adapters for some standard types like `str`, `byte`, `io.IOBase`, `StreamReader` or `DataQueue`. -2. a generator is not supported as data provider anymore, `streamer` can be used instead. - For example, to upload data from file:: +2. a generator is not supported as data provider anymore, `streamer` + can be used instead. For example, to upload data from file:: @aiohttp.streamer def file_sender(writer, file_name=None): @@ -132,13 +143,16 @@ Various 3. `aiohttp.MsgType` dropped, use `aiohttp.WSMsgType` instead. -4. `ClientResponse.url` is an instance of `yarl.URL` class (`url_obj` is deprecated) +4. `ClientResponse.url` is an instance of `yarl.URL` class (`url_obj` + is deprecated) -5. `ClientResponse.raise_for_status()` raises :exc:`aiohttp.ClientResponseError` exception +5. `ClientResponse.raise_for_status()` raises + :exc:`aiohttp.ClientResponseError` exception -6. `ClientResponse.json()` is strict about response's content type. if content type - does not match, it raises :exc:`aiohttp.ClientResponseError` exception. - To disable content type check you can pass ``None`` as `content_type` parameter. +6. `ClientResponse.json()` is strict about response's content type. if + content type does not match, it raises + :exc:`aiohttp.ClientResponseError` exception. To disable content + type check you can pass ``None`` as `content_type` parameter. From 3cfd49c29b3d3fd7211c8f33a75c062421a077ba Mon Sep 17 00:00:00 2001 From: Alex Khomchenko Date: Thu, 3 Aug 2017 01:15:11 -0700 Subject: [PATCH 156/167] Fix link to migration docs (#2160) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2d58ba63c29..47b48c5cc25 100644 --- a/README.rst +++ b/README.rst @@ -42,7 +42,7 @@ organization https://github.com/aio-libs With this amount of api changes we had to make backward incompatible changes. Please check this migration document -http://aiohttp.readthedocs.io/en/latest/migration.html +https://aiohttp.readthedocs.io/en/latest/migration_to_2xx.html Please report problems or annoyance with with api to https://github.com/aio-libs/aiohttp From d67332f7332875df395e93e469fd29e01fcf0a62 Mon Sep 17 00:00:00 2001 From: Mihail Russu Date: Thu, 3 Aug 2017 16:21:54 +0300 Subject: [PATCH 157/167] remove duplicate point (#2162) --- docs/migration_to_2xx.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/migration_to_2xx.rst b/docs/migration_to_2xx.rst index a9883ba84e9..2f3c57ce24a 100644 --- a/docs/migration_to_2xx.rst +++ b/docs/migration_to_2xx.rst @@ -199,8 +199,6 @@ WebRequest and WebResponse 5. `WebSocketResponse.protocol` is renamed to `WebSocketResponse.ws_protocol`. `WebSocketResponse.protocol` is instance of `RequestHandler` class. -5. `WebSocketResponse.protocol` is renamed to `WebSocketResponse.ws_protocol`. - `WebSocketResponse.protocol` is instance of `RequestHandler` class. RequestPayloadError From e9bf20da0c6d3e7ae063c18c6ae072b898b4f7b7 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Thu, 3 Aug 2017 20:16:04 +0200 Subject: [PATCH 158/167] Update pyflakes from 1.5.0 to 1.6.0 (#2163) --- requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index da86e4636df..19eb9dab637 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -3,7 +3,7 @@ setuptools-git==1.2 -r doc.txt pip==9.0.1 flake8==3.4.1 -pyflakes==1.5.0 +pyflakes==1.6.0 coverage==4.4.1 cchardet==2.1.1 cython==0.26 From 8dbad8ccf540cce9c2ed80f8d2ab27c1c202284d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 4 Aug 2017 11:13:01 +0300 Subject: [PATCH 159/167] Route definitions (#2004) * First scratches * Work on * Work on decorators * Make examples work * Refactor * sort modules for scanning * Go forward * Add tests * Add test for decoration methods * Add missing file * Fix python 3.4, add test * Fix typo * Implement RouteDef * Test cover * RouteDef -> RoutesDef * RouteInfo -> RouteDef * Add couple TODOs, drop RouteDef from exported names * Fix flake8 blame * RoutesDef -> RouteTableDef * Add reprs * Add changes record * Test cover missed case * Add documentation for new route definitions API in web reference * Fix typo * Mention route tables and route decorators in web usage * Text flow polishing * Fix typo --- aiohttp/web_urldispatcher.py | 107 +++++++++++- changes/2004.feature | 1 + docs/web.rst | 189 ++++++++++++++------- docs/web_reference.rst | 191 ++++++++++++++++++++- examples/web_srv_route_deco.py | 63 +++++++ examples/web_srv_route_table.py | 62 +++++++ tests/test_route_def.py | 286 ++++++++++++++++++++++++++++++++ 7 files changed, 833 insertions(+), 66 deletions(-) create mode 100644 changes/2004.feature create mode 100644 examples/web_srv_route_deco.py create mode 100644 examples/web_srv_route_table.py create mode 100644 tests/test_route_def.py diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 9fc9cc506f5..d4e6845118d 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -6,7 +6,8 @@ import os import re import warnings -from collections.abc import Container, Iterable, Sized +from collections import namedtuple +from collections.abc import Container, Iterable, Sequence, Sized from functools import wraps from pathlib import Path from types import MappingProxyType @@ -28,13 +29,32 @@ __all__ = ('UrlDispatcher', 'UrlMappingMatchInfo', 'AbstractResource', 'Resource', 'PlainResource', 'DynamicResource', 'AbstractRoute', 'ResourceRoute', - 'StaticResource', 'View') + 'StaticResource', 'View', 'RouteDef', 'RouteTableDef', + 'head', 'get', 'post', 'patch', 'put', 'delete', 'route') HTTP_METHOD_RE = re.compile(r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$") ROUTE_RE = re.compile(r'(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})') PATH_SEP = re.escape('/') +class RouteDef(namedtuple('_RouteDef', 'method, path, handler, kwargs')): + def __repr__(self): + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(", {}={!r}".format(name, value)) + return (" {handler.__name__!r}" + "{info}>".format(method=self.method, path=self.path, + handler=self.handler, info=''.join(info))) + + def register(self, router): + if self.method in hdrs.METH_ALL: + reg = getattr(router, 'add_'+self.method.lower()) + reg(self.path, self.handler, **self.kwargs) + else: + router.add_route(self.method, self.path, self.handler, + **self.kwargs) + + class AbstractResource(Sized, Iterable): def __init__(self, *, name=None): @@ -897,3 +917,86 @@ def freeze(self): super().freeze() for resource in self._resources: resource.freeze() + + def add_routes(self, routes): + """Append routes to route table. + + Parameter should be a sequence of RouteDef objects. + """ + # TODO: add_table maybe? + for route in routes: + route.register(self) + + +def route(method, path, handler, **kwargs): + return RouteDef(method, path, handler, kwargs) + + +def head(path, handler, **kwargs): + return route(hdrs.METH_HEAD, path, handler, **kwargs) + + +def get(path, handler, *, name=None, allow_head=True, **kwargs): + return route(hdrs.METH_GET, path, handler, name=name, + allow_head=allow_head, **kwargs) + + +def post(path, handler, **kwargs): + return route(hdrs.METH_POST, path, handler, **kwargs) + + +def put(path, handler, **kwargs): + return route(hdrs.METH_PUT, path, handler, **kwargs) + + +def patch(path, handler, **kwargs): + return route(hdrs.METH_PATCH, path, handler, **kwargs) + + +def delete(path, handler, **kwargs): + return route(hdrs.METH_DELETE, path, handler, **kwargs) + + +class RouteTableDef(Sequence): + """Route definition table""" + def __init__(self): + self._items = [] + + def __repr__(self): + return "".format(len(self._items)) + + def __getitem__(self, index): + return self._items[index] + + def __iter__(self): + return iter(self._items) + + def __len__(self): + return len(self._items) + + def __contains__(self, item): + return item in self._items + + def route(self, method, path, **kwargs): + def inner(handler): + self._items.append(RouteDef(method, path, handler, kwargs)) + return handler + return inner + + def head(self, path, **kwargs): + return self.route(hdrs.METH_HEAD, path, **kwargs) + + def get(self, path, **kwargs): + return self.route(hdrs.METH_GET, path, **kwargs) + + def post(self, path, **kwargs): + return self.route(hdrs.METH_POST, path, **kwargs) + + def put(self, path, **kwargs): + return self.route(hdrs.METH_PUT, path, **kwargs) + + def patch(self, path, **kwargs): + return self.route(hdrs.METH_PATCH, path, **kwargs) + + def delete(self, path, **kwargs): + return self.route(hdrs.METH_DELETE, path, **kwargs) diff --git a/changes/2004.feature b/changes/2004.feature new file mode 100644 index 00000000000..b6f51e64037 --- /dev/null +++ b/changes/2004.feature @@ -0,0 +1 @@ +Implement `router.add_routes` and router decorators. diff --git a/docs/web.rst b/docs/web.rst index 0b6803302ca..027a4df2736 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -151,17 +151,6 @@ family are plain shortcuts for :meth:`UrlDispatcher.add_route`. Introduce resources. -.. _aiohttp-web-custom-resource: - -Custom resource implementation ------------------------------- - -To register custom resource use :meth:`UrlDispatcher.register_resource`. -Resource instance must implement `AbstractResource` interface. - -.. versionadded:: 1.2.1 - - .. _aiohttp-web-variable-handler: Variable Resources @@ -331,6 +320,69 @@ viewed using the :meth:`UrlDispatcher.named_resources` method:: :meth:`UrlDispatcher.resources` instead of :meth:`UrlDispatcher.named_routes` / :meth:`UrlDispatcher.routes`. + +Alternative ways for registering routes +--------------------------------------- + +Code examples shown above use *imperative* style for adding new +routes: they call ``app.router.add_get(...)`` etc. + +There are two alternatives: route tables and route decorators. + +Route tables look like Django way:: + + async def handle_get(request): + ... + + + async def handle_post(request): + ... + + app.router.add_routes([web.get('/get', handle_get), + web.post('/post', handle_post), + + +The snippet calls :meth:`~aiohttp.web.UrlDispather.add_routes` to +register a list of *route definitions* (:class:`aiohttp.web.RouteDef` +instances) created by :func:`aiohttp.web.get` or +:func:`aiohttp.web.post` functions. + +.. seealso:: :ref:`aiohttp-web-route-def` reference. + +Route decorators are closer to Flask approach:: + + routes = web.RouteTableDef() + + @routes.get('/get') + async def handle_get(request): + ... + + + @routes.post('/post') + async def handle_post(request): + ... + + app.router.add_routes(routes) + +The example creates a :class:`aiohttp.web.RouteTableDef` container first. + +The container is a list-like object with additional decorators +:meth:`aiohttp.web.RouteTableDef.get`, +:meth:`aiohttp.web.RouteTableDef.post` etc. for registering new +routes. + +After filling the container +:meth:`~aiohttp.web.UrlDispather.add_routes` is used for adding +registered *route definitions* into application's router. + +.. seealso:: :ref:`aiohttp-web-route-table-def` reference. + +All tree ways (imperative calls, route tables and decorators) are +equivalent, you could use what do you prefer or even mix them on your +own. + +.. versionadded:: 2.3 + Custom Routing Criteria ----------------------- @@ -483,58 +535,6 @@ third-party library, :mod:`aiohttp_session`, that adds *session* support:: web.run_app(make_app()) -.. _aiohttp-web-expect-header: - -*Expect* Header ---------------- - -:mod:`aiohttp.web` supports *Expect* header. By default it sends -``HTTP/1.1 100 Continue`` line to client, or raises -:exc:`HTTPExpectationFailed` if header value is not equal to -"100-continue". It is possible to specify custom *Expect* header -handler on per route basis. This handler gets called if *Expect* -header exist in request after receiving all headers and before -processing application's :ref:`aiohttp-web-middlewares` and -route handler. Handler can return *None*, in that case the request -processing continues as usual. If handler returns an instance of class -:class:`StreamResponse`, *request handler* uses it as response. Also -handler can raise a subclass of :exc:`HTTPException`. In this case all -further processing will not happen and client will receive appropriate -http response. - -.. note:: - A server that does not understand or is unable to comply with any of the - expectation values in the Expect field of a request MUST respond with - appropriate error status. The server MUST respond with a 417 - (Expectation Failed) status if any of the expectations cannot be met or, - if there are other problems with the request, some other 4xx status. - - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20 - -If all checks pass, the custom handler *must* write a *HTTP/1.1 100 Continue* -status code before returning. - -The following example shows how to setup a custom handler for the *Expect* -header:: - - async def check_auth(request): - if request.version != aiohttp.HttpVersion11: - return - - if request.headers.get('EXPECT') != '100-continue': - raise HTTPExpectationFailed(text="Unknown Expect: %s" % expect) - - if request.headers.get('AUTHORIZATION') is None: - raise HTTPForbidden() - - request.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n") - - async def hello(request): - return web.Response(body=b"Hello, world") - - app = web.Application() - app.router.add_get('/', hello, expect_handler=check_auth) - .. _aiohttp-web-forms: HTTP Forms @@ -1108,6 +1108,69 @@ To manual mode switch :meth:`~StreamResponse.set_tcp_cork` and be helpful for better streaming control for example. +.. _aiohttp-web-expect-header: + +*Expect* Header +--------------- + +:mod:`aiohttp.web` supports *Expect* header. By default it sends +``HTTP/1.1 100 Continue`` line to client, or raises +:exc:`HTTPExpectationFailed` if header value is not equal to +"100-continue". It is possible to specify custom *Expect* header +handler on per route basis. This handler gets called if *Expect* +header exist in request after receiving all headers and before +processing application's :ref:`aiohttp-web-middlewares` and +route handler. Handler can return *None*, in that case the request +processing continues as usual. If handler returns an instance of class +:class:`StreamResponse`, *request handler* uses it as response. Also +handler can raise a subclass of :exc:`HTTPException`. In this case all +further processing will not happen and client will receive appropriate +http response. + +.. note:: + A server that does not understand or is unable to comply with any of the + expectation values in the Expect field of a request MUST respond with + appropriate error status. The server MUST respond with a 417 + (Expectation Failed) status if any of the expectations cannot be met or, + if there are other problems with the request, some other 4xx status. + + http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20 + +If all checks pass, the custom handler *must* write a *HTTP/1.1 100 Continue* +status code before returning. + +The following example shows how to setup a custom handler for the *Expect* +header:: + + async def check_auth(request): + if request.version != aiohttp.HttpVersion11: + return + + if request.headers.get('EXPECT') != '100-continue': + raise HTTPExpectationFailed(text="Unknown Expect: %s" % expect) + + if request.headers.get('AUTHORIZATION') is None: + raise HTTPForbidden() + + request.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n") + + async def hello(request): + return web.Response(body=b"Hello, world") + + app = web.Application() + app.router.add_get('/', hello, expect_handler=check_auth) + +.. _aiohttp-web-custom-resource: + +Custom resource implementation +------------------------------ + +To register custom resource use :meth:`UrlDispatcher.register_resource`. +Resource instance must implement `AbstractResource` interface. + +.. versionadded:: 1.2.1 + + .. _aiohttp-web-graceful-shutdown: Graceful shutdown diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 1a25f0b5190..84db0e1b226 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -20,7 +20,7 @@ Servers` (which have no applications, routers, signals and middlewares) and :class:`Request` has an *application* and *match info* attributes. -A :class:`BaseRequest`/:class:`Request` are :obj:`dict`-like objects, +A :class:`BaseRequest` / :class:`Request` are :obj:`dict` like objects, allowing them to be used for :ref:`sharing data` among :ref:`aiohttp-web-middlewares` and :ref:`aiohttp-web-signals` handlers. @@ -1511,6 +1511,15 @@ Router is any object that implements :class:`AbstractRouter` interface. :returns: new :class:`PlainRoute` or :class:`DynamicRoute` instance. + .. method:: add_routes(routes_table) + + Register route definitions from *routes_table*. + + The table is a :class:`list` of :class:`RouteDef` items or + :class:`RouteTableDef`. + + .. versionadded:: 2.3 + .. method:: add_get(path, handler, *, name=None, allow_head=True, **kwargs) Shortcut for adding a GET handler. Calls the :meth:`add_route` with \ @@ -2019,6 +2028,186 @@ and *405 Method Not Allowed*. HTTP status reason +.. _aiohttp-web-route-def: + + +RouteDef +^^^^^^^^ + +Route definition, a description for not registered yet route. + +Could be used for filing route table by providing a list of route +definitions (Django style). + +The definition is created by functions like :func:`get` or +:func:`post`, list of definitions could be added to router by +:meth:`UrlDispatcher.add_routes` call:: + + from aiohttp import web + + async def handle_get(request): + ... + + + async def handle_post(request): + ... + + app.router.add_routes([web.get('/get', handle_get), + web.post('/post', handle_post), + + +.. class:: RouteDef + + A definition for not added yet route. + + .. attribute:: method + + HTTP method (``GET``, ``POST`` etc.) (:class:`str`). + + .. attribute:: path + + Path to resource, e.g. ``/path/to``. Could contain ``{}`` + brackets for :ref:`variable resources + ` (:class:`str`). + + .. attribute:: handler + + An async function to handle HTTP request. + + .. attribute:: kwargs + + A :class:`dict` of additional arguments. + + .. versionadded:: 2.3 + + +.. function:: get(path, handler, *, name=None, allow_head=True, \ + expect_handler=None) + + Return :class:`RouteDef` for processing ``GET`` requests. See + :meth:`UrlDispatcher.add_get` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: post(path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``POST`` requests. See + :meth:`UrlDispatcher.add_post` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: head(path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``HEAD`` requests. See + :meth:`UrlDispatcher.add_head` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: put(path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``PUT`` requests. See + :meth:`UrlDispatcher.add_put` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: patch(path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``PATCH`` requests. See + :meth:`UrlDispatcher.add_patch` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: delete(path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``DELETE`` requests. See + :meth:`UrlDispatcher.add_delete` for information about parameters. + + .. versionadded:: 2.3 + +.. function:: route(method, path, handler, *, name=None, expect_handler=None) + + Return :class:`RouteDef` for processing ``POST`` requests. See + :meth:`UrlDispatcher.add_route` for information about parameters. + + .. versionadded:: 2.3 + +.. _aiohttp-web-route-table-def: + +RouteTableDef +^^^^^^^^^^^^^ + +A routes table definition used for describing routes by decorators +(Flask style):: + + from aiohttp import web + + routes = web.RouteTableDef() + + @routes.get('/get') + async def handle_get(request): + ... + + + @routes.post('/post') + async def handle_post(request): + ... + + app.router.add_routes(routes) + +.. class:: RouteTableDef() + + A sequence of :class:`RouteDef` instances (implements + :class:`abc.collections.Sequence` protocol). + + In addition to all standard :class:`list` methods the class + provides also methods like ``get()`` and ``post()`` for adding new + route definition. + + .. decoratormethod:: get(path, *, allow_head=True, \ + name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``GET`` web-handler. + + See :meth:`UrlDispatcher.add_get` for information about parameters. + + .. decoratormethod:: post(path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``POST`` web-handler. + + See :meth:`UrlDispatcher.add_post` for information about parameters. + + .. decoratormethod:: head(path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``HEAD`` web-handler. + + See :meth:`UrlDispatcher.add_head` for information about parameters. + + .. decoratormethod:: put(path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``PUT`` web-handler. + + See :meth:`UrlDispatcher.add_put` for information about parameters. + + .. decoratormethod:: patch(path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``PATCH`` web-handler. + + See :meth:`UrlDispatcher.add_patch` for information about parameters. + + .. decoratormethod:: delete(path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering ``DELETE`` web-handler. + + See :meth:`UrlDispatcher.add_delete` for information about parameters. + + .. decoratormethod:: route(method, path, *, name=None, expect_handler=None) + + Add a new :class:`RouteDef` item for registering a web-handler + for arbitrary HTTP method. + + See :meth:`UrlDispatcher.add_route` for information about parameters. + + .. versionadded:: 2.3 MatchInfo ^^^^^^^^^ diff --git a/examples/web_srv_route_deco.py b/examples/web_srv_route_deco.py new file mode 100644 index 00000000000..9accbf6e386 --- /dev/null +++ b/examples/web_srv_route_deco.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Example for aiohttp.web basic server +with decorator definition for routes +""" + +import asyncio +import textwrap + +from aiohttp import web + + +routes = web.RouteTableDef() + + +@routes.get('/') +async def intro(request): + txt = textwrap.dedent("""\ + Type {url}/hello/John {url}/simple or {url}/change_body + in browser url bar + """).format(url='127.0.0.1:8080') + binary = txt.encode('utf8') + resp = web.StreamResponse() + resp.content_length = len(binary) + resp.content_type = 'text/plain' + await resp.prepare(request) + resp.write(binary) + return resp + + +@routes.get('/simple') +async def simple(request): + return web.Response(text="Simple answer") + + +@routes.get('/change_body') +async def change_body(request): + resp = web.Response() + resp.body = b"Body changed" + resp.content_type = 'text/plain' + return resp + + +@routes.get('/hello') +async def hello(request): + resp = web.StreamResponse() + name = request.match_info.get('name', 'Anonymous') + answer = ('Hello, ' + name).encode('utf8') + resp.content_length = len(answer) + resp.content_type = 'text/plain' + await resp.prepare(request) + resp.write(answer) + await resp.write_eof() + return resp + + +async def init(): + app = web.Application() + app.router.add_routes(routes) + return app + +loop = asyncio.get_event_loop() +app = loop.run_until_complete(init()) +web.run_app(app) diff --git a/examples/web_srv_route_table.py b/examples/web_srv_route_table.py new file mode 100644 index 00000000000..7d8af62a5c2 --- /dev/null +++ b/examples/web_srv_route_table.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Example for aiohttp.web basic server +with table definition for routes +""" + +import asyncio +import textwrap + +from aiohttp import web + + +async def intro(request): + txt = textwrap.dedent("""\ + Type {url}/hello/John {url}/simple or {url}/change_body + in browser url bar + """).format(url='127.0.0.1:8080') + binary = txt.encode('utf8') + resp = web.StreamResponse() + resp.content_length = len(binary) + resp.content_type = 'text/plain' + await resp.prepare(request) + resp.write(binary) + return resp + + +async def simple(request): + return web.Response(text="Simple answer") + + +async def change_body(request): + resp = web.Response() + resp.body = b"Body changed" + resp.content_type = 'text/plain' + return resp + + +async def hello(request): + resp = web.StreamResponse() + name = request.match_info.get('name', 'Anonymous') + answer = ('Hello, ' + name).encode('utf8') + resp.content_length = len(answer) + resp.content_type = 'text/plain' + await resp.prepare(request) + resp.write(answer) + await resp.write_eof() + return resp + + +async def init(): + app = web.Application() + app.router.add_routes([ + web.get('/', intro), + web.get('/simple', simple), + web.get('/change_body', change_body), + web.get('/hello/{name}', hello), + web.get('/hello', hello), + ]) + return app + +loop = asyncio.get_event_loop() +app = loop.run_until_complete(init()) +web.run_app(app) diff --git a/tests/test_route_def.py b/tests/test_route_def.py new file mode 100644 index 00000000000..730f73da6a7 --- /dev/null +++ b/tests/test_route_def.py @@ -0,0 +1,286 @@ +import asyncio + +import pytest + +from aiohttp import web +from aiohttp.web_urldispatcher import UrlDispatcher + + +@pytest.fixture +def router(): + return UrlDispatcher() + + +def test_get(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.get('/', handler)]) + assert len(router.routes()) == 2 # GET and HEAD + + route = list(router.routes())[1] + assert route.handler is handler + assert route.method == 'GET' + assert str(route.url_for()) == '/' + + route2 = list(router.routes())[0] + assert route2.handler is handler + assert route2.method == 'HEAD' + + +def test_head(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.head('/', handler)]) + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'HEAD' + assert str(route.url_for()) == '/' + + +def test_post(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.post('/', handler)]) + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'POST' + assert str(route.url_for()) == '/' + + +def test_put(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.put('/', handler)]) + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'PUT' + assert str(route.url_for()) == '/' + + +def test_patch(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.patch('/', handler)]) + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'PATCH' + assert str(route.url_for()) == '/' + + +def test_delete(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.delete('/', handler)]) + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'DELETE' + assert str(route.url_for()) == '/' + + +def test_route(router): + @asyncio.coroutine + def handler(request): + pass + + router.add_routes([web.route('OTHER', '/', handler)]) + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.handler is handler + assert route.method == 'OTHER' + assert str(route.url_for()) == '/' + + +def test_head_deco(router): + routes = web.RouteTableDef() + + @routes.head('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'HEAD' + assert str(route.url_for()) == '/path' + + +def test_get_deco(router): + routes = web.RouteTableDef() + + @routes.get('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 2 + + route1 = list(router.routes())[0] + assert route1.method == 'HEAD' + assert str(route1.url_for()) == '/path' + + route2 = list(router.routes())[1] + assert route2.method == 'GET' + assert str(route2.url_for()) == '/path' + + +def test_post_deco(router): + routes = web.RouteTableDef() + + @routes.post('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'POST' + assert str(route.url_for()) == '/path' + + +def test_put_deco(router): + routes = web.RouteTableDef() + + @routes.put('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'PUT' + assert str(route.url_for()) == '/path' + + +def test_patch_deco(router): + routes = web.RouteTableDef() + + @routes.patch('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'PATCH' + assert str(route.url_for()) == '/path' + + +def test_delete_deco(router): + routes = web.RouteTableDef() + + @routes.delete('/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'DELETE' + assert str(route.url_for()) == '/path' + + +def test_route_deco(router): + routes = web.RouteTableDef() + + @routes.route('OTHER', '/path') + @asyncio.coroutine + def handler(request): + pass + + router.add_routes(routes) + + assert len(router.routes()) == 1 + + route = list(router.routes())[0] + assert route.method == 'OTHER' + assert str(route.url_for()) == '/path' + + +def test_routedef_sequence_protocol(): + routes = web.RouteTableDef() + + @routes.delete('/path') + @asyncio.coroutine + def handler(request): + pass + + assert len(routes) == 1 + + info = routes[0] + assert isinstance(info, web.RouteDef) + assert info in routes + assert list(routes)[0] is info + + +def test_repr_route_def(): + routes = web.RouteTableDef() + + @routes.get('/path') + @asyncio.coroutine + def handler(request): + pass + + rd = routes[0] + assert repr(rd) == " 'handler'>" + + +def test_repr_route_def_with_extra_info(): + routes = web.RouteTableDef() + + @routes.get('/path', extra='info') + @asyncio.coroutine + def handler(request): + pass + + rd = routes[0] + assert repr(rd) == " 'handler', extra='info'>" + + +def test_repr_route_table_def(): + routes = web.RouteTableDef() + + @routes.get('/path') + @asyncio.coroutine + def handler(request): + pass + + assert repr(routes) == "" From 4aba753a1635ee1d28766611088004128d18ed13 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 5 Aug 2017 01:02:33 +0300 Subject: [PATCH 160/167] Fix #1909: add example for middleware execution order (#2165) --- docs/web.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/web.rst b/docs/web.rst index 027a4df2736..5a5170485b7 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -922,6 +922,43 @@ if user has no permissions to access the underlying resource. They may also render errors raised by the handler, perform some pre- or post-processing like handling *CORS* and so on. +The following code demonstrates middlewares execution order:: + + from aiohttp import web + def test(request): + print('Handler function called') + return web.Response(text="Hello") + + async def middleware1(app, handler): + async def middleware_handler(request): + print('Middleware 1 called') + response = await handler(request) + print('Middleware 1 finished') + + return response + return middleware_handler + + async def middleware2(app, handler): + async def middleware_handler(request): + print('Middleware 2 called') + response = await handler(request) + print('Middleware 2 finished') + + return response + return middleware_handler + + + app = web.Application(middlewares=[middleware1, middleware2]) + app.router.add_get('/', test) + web.run_app(app) + +Produced output:: + + Middleware 1 called + Middleware 2 called + Handler function called + Middleware 2 finished + Middleware 1 finished Example ^^^^^^^ From ee69c4026ae035bd0a488cbe718eaae5d49fc0e1 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 5 Aug 2017 17:14:43 +0300 Subject: [PATCH 161/167] add append_version feature into StaticResource url resolver (#2158) * add StaticResource.url append_version arg * add 2157.feature * 2157.feature contributor * sort contributors.txt * add documentation for StaticResource.url_for append_version arg #2157 * update read file code to be compatible with windows python 3.4 * fix flake8 line len issues * add append_version param to StaticResource constructor and add_static method docs and tests is updated to reflect changes * add more tests and docs for StaticRoute append_version * removed outdated comments * add ValueError exception catching for follow_symlinks and more tests --- CONTRIBUTORS.txt | 1 + aiohttp/web_urldispatcher.py | 56 ++++++++++++++--- changes/2157.feature | 2 + docs/web.rst | 11 ++++ docs/web_reference.rst | 19 +++++- tests/test_urldispatch.py | 119 +++++++++++++++++++++++++++++++++++ 6 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 changes/2157.feature diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index f2deee8a68a..fe67c5e800d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -79,6 +79,7 @@ Günther Jena Hu Bo Hugo Herter Hynek Schlawack +Igor Alexandrov Igor Davydenko Igor Pavlov Ingmar Steen diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index d4e6845118d..3793ec946a0 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -1,6 +1,8 @@ import abc import asyncio +import base64 import collections +import hashlib import inspect import keyword import os @@ -442,11 +444,13 @@ def add_prefix(self, prefix): class StaticResource(PrefixResource): + VERSION_KEY = 'v' def __init__(self, prefix, directory, *, name=None, - expect_handler=None, chunk_size=256*1024, + expect_handler=None, chunk_size=256 * 1024, response_factory=StreamResponse, - show_index=False, follow_symlinks=False): + show_index=False, follow_symlinks=False, + append_version=False): super().__init__(prefix, name=name) try: directory = Path(directory) @@ -463,6 +467,7 @@ def __init__(self, prefix, directory, *, name=None, self._chunk_size = chunk_size self._follow_symlinks = follow_symlinks self._expect_handler = expect_handler + self._append_version = append_version self._routes = {'GET': ResourceRoute('GET', self._handle, self, expect_handler=expect_handler), @@ -470,17 +475,48 @@ def __init__(self, prefix, directory, *, name=None, 'HEAD': ResourceRoute('HEAD', self._handle, self, expect_handler=expect_handler)} - def url(self, *, filename, query=None): - return str(self.url_for(filename=filename).with_query(query)) + def url(self, *, filename, append_version=None, query=None): + url = self.url_for(filename=filename, append_version=append_version) + if query is not None: + return str(url.update_query(query)) + return str(url) - def url_for(self, *, filename): + def url_for(self, *, filename, append_version=None): + if append_version is None: + append_version = self._append_version if isinstance(filename, Path): filename = str(filename) while filename.startswith('/'): filename = filename[1:] filename = '/' + filename url = self._prefix + URL(filename).raw_path - return URL(url) + url = URL(url) + if append_version is True: + try: + if filename.startswith('/'): + filename = filename[1:] + filepath = self._directory.joinpath(filename).resolve() + if not self._follow_symlinks: + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink + # with follow_symlinks is False + return url # relatively safe + if filepath.is_file(): + # TODO cache file content + # with file watcher for cache invalidation + with open(str(filepath), mode='rb') as f: + file_bytes = f.read() + h = self._get_file_hash(file_bytes) + url = url.with_query({self.VERSION_KEY: h}) + return url + return url + + def _get_file_hash(self, byte_array): + m = hashlib.sha256() # todo sha256 can be configurable param + m.update(byte_array) + b64 = base64.urlsafe_b64encode(m.digest()) + return b64.decode('ascii') def get_info(self): return {'directory': self._directory, @@ -848,8 +884,9 @@ def add_route(self, method, path, handler, expect_handler=expect_handler) def add_static(self, prefix, path, *, name=None, expect_handler=None, - chunk_size=256*1024, response_factory=StreamResponse, - show_index=False, follow_symlinks=False): + chunk_size=256 * 1024, response_factory=StreamResponse, + show_index=False, follow_symlinks=False, + append_version=False): """Add static files view. prefix - url prefix @@ -865,7 +902,8 @@ def add_static(self, prefix, path, *, name=None, expect_handler=None, chunk_size=chunk_size, response_factory=response_factory, show_index=show_index, - follow_symlinks=follow_symlinks) + follow_symlinks=follow_symlinks, + append_version=append_version) self.register_resource(resource) return resource diff --git a/changes/2157.feature b/changes/2157.feature new file mode 100644 index 00000000000..ce3e362e43f --- /dev/null +++ b/changes/2157.feature @@ -0,0 +1,2 @@ +add `append_version` arg into `StaticResource.url` and `StaticResource.url_for` methods +for getting an url with hash (version) of the file. \ No newline at end of file diff --git a/docs/web.rst b/docs/web.rst index 5a5170485b7..76300f29a49 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -455,6 +455,17 @@ symlinks, parameter ``follow_symlinks`` should be set to ``True``:: app.router.add_static('/prefix', path_to_static_folder, follow_symlinks=True) +When you want to enable cache busting, +parameter ``append_version`` can be set to ``True`` + +Cache busting is the process of appending some form of file version hash +to the filename of resources like JavaScript and CSS files. +The performance advantage of doing this is that we can tell the browser +to cache these files indefinitely without worrying about the client not getting +the latest version when the file changes:: + + app.router.add_static('/prefix', path_to_static_folder, append_version=True) + Template Rendering ------------------ diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 84db0e1b226..6fb18997e29 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1580,7 +1580,8 @@ Router is any object that implements :class:`AbstractRouter` interface. chunk_size=256*1024, \ response_factory=StreamResponse, \ show_index=False, \ - follow_symlinks=False) + follow_symlinks=False, \ + append_version=False) Adds a router and a handler for returning static files. @@ -1646,6 +1647,12 @@ Router is any object that implements :class:`AbstractRouter` interface. a directory, by default it's not allowed and HTTP/404 will be returned on access. + :param bool append_version: flag for adding file version (hash) + to the url query string, this value will be used + as default when you call to :meth:`StaticRoute.url` + and :meth:`StaticRoute.url_for` methods. + + :returns: new :class:`StaticRoute` instance. .. method:: add_subapp(prefix, subapp) @@ -1933,7 +1940,7 @@ Resource classes hierarchy:: The class corresponds to resources for :ref:`static file serving `. - .. method:: url_for(filename) + .. method:: url_for(filename, append_version=None) Returns a :class:`~yarl.URL` for file path under resource prefix. @@ -1944,6 +1951,14 @@ Resource classes hierarchy:: E.g. an URL for ``'/prefix/dir/file.txt'`` should be generated as ``resource.url_for(filename='dir/file.txt')`` + :param bool append_version: -- a flag for adding file version (hash) to the url query string for cache boosting + + By default has value from an constructor (``False`` by default) + When set to ``True`` - ``v=FILE_HASH`` query string param will be added + When set to ``False`` has no impact + + if file not found has no impact + .. versionadded:: 1.1 .. class:: PrefixedSubAppResource diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index 8f5eae5af43..00a317fd7cb 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -377,6 +377,125 @@ def test_add_static(router): assert len(resource) == 2 +def test_add_static_append_version(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/data.unknown_mime_type', append_version=True) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_set_from_constructor(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + append_version=True, + name='static') + url = resource.url(filename='/data.unknown_mime_type') + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_override_constructor(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + append_version=True, + name='static') + url = resource.url(filename='/data.unknown_mime_type', + append_version=False) + expect_url = '/st/data.unknown_mime_type' + assert expect_url == url + + +def test_add_static_append_version_filename_without_slash(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='data.unknown_mime_type', append_version=True) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_non_exists_file(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/non_exists_file', append_version=True) + assert '/st/non_exists_file' == url + + +def test_add_static_append_version_non_exists_file_without_slash(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='non_exists_file', append_version=True) + assert '/st/non_exists_file' == url + + +def test_add_static_append_version_follow_symlink(router, tmpdir): + """ + Tests the access to a symlink, in static folder with apeend_version + """ + tmp_dir_path = str(tmpdir) + symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink') + symlink_target_path = os.path.dirname(__file__) + os.symlink(symlink_target_path, symlink_path, True) + + # Register global static route: + resource = router.add_static('/st', tmp_dir_path, follow_symlinks=True, + append_version=True) + + url = resource.url( + filename='/append_version_symlink/data.unknown_mime_type') + + expect_url = '/st/append_version_symlink/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_not_follow_symlink(router, tmpdir): + """ + Tests the access to a symlink, in static folder with apeend_version + """ + tmp_dir_path = str(tmpdir) + symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink') + symlink_target_path = os.path.dirname(__file__) + os.symlink(symlink_target_path, symlink_path, True) + + # Register global static route: + resource = router.add_static('/st', tmp_dir_path, follow_symlinks=False, + append_version=True) + + filename = '/append_version_symlink/data.unknown_mime_type' + url = resource.url(filename=filename) + assert '/st/append_version_symlink/data.unknown_mime_type' == url + + +def test_add_static_append_version_with_query(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/data.unknown_mime_type', + append_version=True, + query={'key': 'val'}) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D&key=val' + assert expect_url == url + + +def test_add_static_append_version_non_exists_file_with_query(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/non_exists_file', + append_version=True, + query={'key': 'val'}) + assert '/st/non_exists_file?key=val' == url + + def test_plain_not_match(router): handler = make_handler() router.add_route('GET', '/get/path', handler, name='name') From bdd862b47b204bdfa4cad02e21d871020b994255 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Sun, 6 Aug 2017 20:35:45 +0100 Subject: [PATCH 162/167] add helpmanual.io to powered_by --- docs/powered_by.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/powered_by.rst b/docs/powered_by.rst index c1daac4fade..588d609bf50 100644 --- a/docs/powered_by.rst +++ b/docs/powered_by.rst @@ -18,3 +18,4 @@ make a Pull Request! * `Morpheus messaging microservice `_ * `Eyepea - Custom telephony solutions `_ * `ALLOcloud - Telephony in the cloud `_ +* `helpmanual - comprehensive help and man page database `_ From dda632bef434628b369fd94097c888b41a104cf7 Mon Sep 17 00:00:00 2001 From: David Poirier Date: Tue, 8 Aug 2017 04:29:46 +1000 Subject: [PATCH 163/167] Deprecated BaseRequest.has_body, replaced with 2 new attributes (#2005) (#2169) * Deprecated BaseRequest.has_body, replaced with 2 new attributes * Test obsolete BaseRequest.has_body attr --- aiohttp/web_request.py | 16 +++++++++++++++- changes/2005.feature | 3 +++ docs/web_reference.rst | 24 +++++++++++++++++++++--- tests/test_web_functional.py | 4 ++++ 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 changes/2005.feature diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index d59c3a61c5a..9a6d9ebf0f0 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -16,6 +16,7 @@ from . import hdrs, multipart from .helpers import HeadersMixin, SimpleCookie, reify, sentinel +from .streams import EmptyStreamReader from .web_exceptions import HTTPRequestEntityTooLarge @@ -457,9 +458,22 @@ def content(self): @property def has_body(self): - """Return True if request has HTTP BODY, False otherwise.""" + """Return True if request's HTTP BODY can be read, False otherwise.""" + warnings.warn( + "Deprecated, use .can_read_body #2005", + DeprecationWarning, stacklevel=2) + return not self._payload.at_eof() + + @property + def can_read_body(self): + """Return True if request's HTTP BODY can be read, False otherwise.""" return not self._payload.at_eof() + @property + def body_exists(self): + """Return True if request has HTTP BODY, False otherwise.""" + return type(self._payload) is not EmptyStreamReader + @asyncio.coroutine def release(self): """Release request. diff --git a/changes/2005.feature b/changes/2005.feature new file mode 100644 index 00000000000..77946cf4db6 --- /dev/null +++ b/changes/2005.feature @@ -0,0 +1,3 @@ +- Deprecated BaseRequest.has_body in favour of BaseRequest.can_read_body +- Added BaseRequest.body_exists attribute that stays static for the lifetime +of the request diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 6fb18997e29..a98929b7199 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -230,15 +230,33 @@ and :ref:`aiohttp-web-signals` handlers. Read-only property. - .. attribute:: has_body + .. attribute:: body_exists Return ``True`` if request has *HTTP BODY*, ``False`` otherwise. Read-only :class:`bool` property. - .. versionadded:: 0.16 + .. versionadded:: 2.3 + + .. attribute:: can_read_body - .. attribute:: content_type + Return ``True`` if request's *HTTP BODY* can be read, ``False`` otherwise. + + Read-only :class:`bool` property. + + .. versionadded:: 2.3 + + .. attribute:: has_body + + Return ``True`` if request's *HTTP BODY* can be read, ``False`` otherwise. + + Read-only :class:`bool` property. + + .. deprecated:: 2.3 + + Use :meth:`can_read_body` instead. + + .. attribute:: content_type Read-only property with *content* part of *Content-Type* header. diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 3f673c2ce9b..9bb26589119 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -694,6 +694,8 @@ def test_empty_content_for_query_without_body(loop, test_client): @asyncio.coroutine def handler(request): + assert not request.body_exists + assert not request.can_read_body assert not request.has_body return web.Response() @@ -710,6 +712,8 @@ def test_empty_content_for_query_with_body(loop, test_client): @asyncio.coroutine def handler(request): + assert request.body_exists + assert request.can_read_body assert request.has_body body = yield from request.read() return web.Response(body=body) From a0666e896070765f9f1cefae047b830f72b85a74 Mon Sep 17 00:00:00 2001 From: Vasiliy Faronov Date: Wed, 9 Aug 2017 11:25:47 +0400 Subject: [PATCH 164/167] Fix parsing the Forwarded header (#2170) (#2173) This fixes #2170 by parsing Forwarded more carefully, while staying about as fast, simple, and robust in the face of potential injection attacks. (Speed was measured with IPython's %timeit on my laptop on a few typical and pathological header values.) In particular: - commas and semicolons are allowed inside quoted-strings; - empty forwarded-pairs (as in "for=_1;;by=_2") are allowed; - non-standard parameters are allowed (although this alone could be easily done in the previous parser). This still doesn't parse valid headers containing obs-text, which was an intentional decision in the previous parser (see comments) that I did not change. Also, the previous parser used to bail out of forwarded-elements containing duplicate parameter names. No rationale was given in the code, and I don't think this is important, so the new parser doesn't enforce this. --- CONTRIBUTORS.txt | 1 + aiohttp/web_request.py | 76 +++++++++++++++++---------------- changes/2170.bugfix | 1 + tests/test_web_request.py | 88 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 39 deletions(-) create mode 100644 changes/2170.bugfix diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index fe67c5e800d..1f50b3224e2 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -172,6 +172,7 @@ Thomas Grainger Tolga Tezel Vaibhav Sagar Vamsi Krishna Avula +Vasiliy Faronov Vasyl Baran Vikas Kawadia Vitalik Verhovodov diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 9a6d9ebf0f0..ffae3b50a62 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -28,7 +28,7 @@ _TCHAR = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" # '-' at the end to prevent interpretation as range in a char class -_TOKEN = r'[{tchar}]*'.format(tchar=_TCHAR) +_TOKEN = r'[{tchar}]+'.format(tchar=_TCHAR) _QDTEXT = r'[{}]'.format( r''.join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F)))) @@ -40,12 +40,8 @@ _QUOTED_STRING = r'"(?:{quoted_pair}|{qdtext})*"'.format( qdtext=_QDTEXT, quoted_pair=_QUOTED_PAIR) -_FORWARDED_PARAMS = ( - r'[bB][yY]|[fF][oO][rR]|[hH][oO][sS][tT]|[pP][rR][oO][tT][oO]') - _FORWARDED_PAIR = ( - r'^({forwarded_params})=({token}|{quoted_string})$'.format( - forwarded_params=_FORWARDED_PARAMS, + r'({token})=({token}|{quoted_string})'.format( token=_TOKEN, quoted_string=_QUOTED_STRING)) @@ -209,37 +205,45 @@ def forwarded(self): Returns a tuple containing one or more immutable dicts """ - def _parse_forwarded(forwarded_headers): - for field_value in forwarded_headers: - # by=...;for=..., For=..., BY=... - for forwarded_elm in field_value.split(','): - # by=...;for=... - fvparams = dict() - forwarded_pairs = ( - _FORWARDED_PAIR_RE.findall(pair) - for pair in forwarded_elm.strip().split(';')) - for forwarded_pair in forwarded_pairs: - # by=... - if len(forwarded_pair) != 1: - # non-compliant syntax - break - param, value = forwarded_pair[0] - if param.lower() in fvparams: - # duplicate param in field-value - break - if value and value[0] == '"': - # quoted string: replace quotes and escape - # sequences - value = _QUOTED_PAIR_REPLACE_RE.sub( - r'\1', value[1:-1]) - fvparams[param.lower()] = value + elems = [] + for field_value in self._message.headers.getall(hdrs.FORWARDED, ()): + length = len(field_value) + pos = 0 + need_separator = False + elem = {} + elems.append(types.MappingProxyType(elem)) + while 0 <= pos < length: + match = _FORWARDED_PAIR_RE.match(field_value, pos) + if match is not None: # got a valid forwarded-pair + if need_separator: + # bad syntax here, skip to next comma + pos = field_value.find(',', pos) else: - yield types.MappingProxyType(fvparams) - continue - yield dict() - - return tuple( - _parse_forwarded(self._message.headers.getall(hdrs.FORWARDED, ()))) + (name, value) = match.groups() + if value[0] == '"': + # quoted string: remove quotes and unescape + value = _QUOTED_PAIR_REPLACE_RE.sub(r'\1', + value[1:-1]) + elem[name.lower()] = value + pos += len(match.group(0)) + need_separator = True + elif field_value[pos] == ',': # next forwarded-element + need_separator = False + elem = {} + elems.append(types.MappingProxyType(elem)) + pos += 1 + elif field_value[pos] == ';': # next forwarded-pair + need_separator = False + pos += 1 + elif field_value[pos] in ' \t': + # Allow whitespace even between forwarded-pairs, though + # RFC 7239 doesn't. This simplifies code and is in line + # with Postel's law. + pos += 1 + else: + # bad syntax here, skip to next comma + pos = field_value.find(',', pos) + return tuple(elems) @reify def _scheme(self): diff --git a/changes/2170.bugfix b/changes/2170.bugfix new file mode 100644 index 00000000000..35961c5651e --- /dev/null +++ b/changes/2170.bugfix @@ -0,0 +1 @@ +Fix parsing the Forwarded header according to RFC 7239. diff --git a/tests/test_web_request.py b/tests/test_web_request.py index e143b4e151b..2ea5655c9c3 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -290,6 +290,61 @@ def test_single_forwarded_header_quoted_escaped(): assert req.forwarded[0]['proto'] == 'lala land~ 123!&' +def test_single_forwarded_header_custom_param(): + header = r'BY=identifier;PROTO=https;SOME="other, \"value\""' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert len(req.forwarded) == 1 + assert req.forwarded[0]['by'] == 'identifier' + assert req.forwarded[0]['proto'] == 'https' + assert req.forwarded[0]['some'] == 'other, "value"' + + +def test_single_forwarded_header_empty_params(): + # This is allowed by the grammar given in RFC 7239 + header = ';For=identifier;;PROTO=https;;;' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['for'] == 'identifier' + assert req.forwarded[0]['proto'] == 'https' + + +def test_single_forwarded_header_bad_separator(): + header = 'BY=identifier PROTO=https' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert 'proto' not in req.forwarded[0] + + +def test_single_forwarded_header_injection1(): + # We might receive a header like this if we're sitting behind a reverse + # proxy that blindly appends a forwarded-element without checking + # the syntax of existing field-values. We should be able to recover + # the appended element anyway. + header = 'for=_injected;by=", for=_real' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert len(req.forwarded) == 2 + assert 'by' not in req.forwarded[0] + assert req.forwarded[1]['for'] == '_real' + + +def test_single_forwarded_header_injection2(): + header = 'very bad syntax, for=_real' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert len(req.forwarded) == 2 + assert 'for' not in req.forwarded[0] + assert req.forwarded[1]['for'] == '_real' + + +def test_single_forwarded_header_long_quoted_string(): + header = 'for="' + '\\\\' * 5000 + '"' + req = make_mocked_request('GET', '/', + headers=CIMultiDict({'Forwarded': header})) + assert req.forwarded[0]['for'] == '\\' * 5000 + + def test_multiple_forwarded_headers(): headers = CIMultiDict() headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') @@ -303,10 +358,37 @@ def test_multiple_forwarded_headers(): assert req.forwarded[2]['for'] == 'identifier5' +def test_multiple_forwarded_headers_bad_syntax(): + headers = CIMultiDict() + headers.add('Forwarded', 'for=_1;by=_2') + headers.add('Forwarded', 'invalid value') + headers.add('Forwarded', '') + headers.add('Forwarded', 'for=_3;by=_4') + req = make_mocked_request('GET', '/', headers=headers) + assert len(req.forwarded) == 4 + assert req.forwarded[0]['for'] == '_1' + assert 'for' not in req.forwarded[1] + assert 'for' not in req.forwarded[2] + assert req.forwarded[3]['by'] == '_4' + + +def test_multiple_forwarded_headers_injection(): + headers = CIMultiDict() + # This could be sent by an attacker, hoping to "shadow" the second header. + headers.add('Forwarded', 'for=_injected;by="') + # This is added by our trusted reverse proxy. + headers.add('Forwarded', 'for=_real;by=_actual_proxy') + req = make_mocked_request('GET', '/', headers=headers) + assert len(req.forwarded) == 2 + assert 'by' not in req.forwarded[0] + assert req.forwarded[1]['for'] == '_real' + assert req.forwarded[1]['by'] == '_actual_proxy' + + def test_https_scheme_by_forwarded_header(): + header = 'by=_1;for=_2;host=_3;proto=https' req = make_mocked_request('GET', '/', - headers=CIMultiDict( - {'Forwarded': 'by=;for=;host=;proto=https'})) + headers=CIMultiDict({'Forwarded': header})) assert "https" == req.scheme assert req.secure is True @@ -338,7 +420,7 @@ def test_https_scheme_by_x_forwarded_proto_header_no_tls(): def test_host_by_forwarded_header(): headers = CIMultiDict() headers.add('Forwarded', 'By=identifier1;for=identifier2, BY=identifier3') - headers.add('Forwarded', 'by=;for=;host=example.com') + headers.add('Forwarded', 'by=unknown;for=unknown;host=example.com') req = make_mocked_request('GET', '/', headers=headers) assert req.host == 'example.com' From c04a75afed8bdd4db68e8ad9fd5765e66f2a2bf9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 9 Aug 2017 10:29:27 +0300 Subject: [PATCH 165/167] Add missing changes record --- changes/2173.feature | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changes/2173.feature diff --git a/changes/2173.feature b/changes/2173.feature new file mode 100644 index 00000000000..d0173ac71dc --- /dev/null +++ b/changes/2173.feature @@ -0,0 +1,8 @@ +Fix parsing the Forwarded header. + +* commas and semicolons are allowed inside quoted-strings; + +* empty forwarded-pairs (as in for=_1;;by=_2) are allowed; + +* non-standard parameters are allowed (although this alone could be + easily done in the previous parser). From 4b498d10d1d4672644932d2848d5d4443395f13d Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Thu, 10 Aug 2017 20:10:50 +0200 Subject: [PATCH 166/167] use trafaret.IP instead (#2187) * use trafaret.IP * Create 2187.misc --- changes/2187.misc | 1 + demos/polls/aiohttpdemo_polls/utils.py | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 changes/2187.misc diff --git a/changes/2187.misc b/changes/2187.misc new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/changes/2187.misc @@ -0,0 +1 @@ + diff --git a/demos/polls/aiohttpdemo_polls/utils.py b/demos/polls/aiohttpdemo_polls/utils.py index 45688130b65..f2c2f309900 100644 --- a/demos/polls/aiohttpdemo_polls/utils.py +++ b/demos/polls/aiohttpdemo_polls/utils.py @@ -1,8 +1,5 @@ import trafaret as T - -primitive_ip_regexp = r'^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' - TRAFARET = T.Dict({ T.Key('postgres'): T.Dict({ @@ -14,6 +11,6 @@ 'minsize': T.Int(), 'maxsize': T.Int(), }), - T.Key('host'): T.String(regex=primitive_ip_regexp), + T.Key('host'): T.IP, T.Key('port'): T.Int(), }) From cd207cfa5a8e38eb7f9af7574b58a885245a0382 Mon Sep 17 00:00:00 2001 From: Kolokotronis Panagiotis Date: Sun, 13 Aug 2017 10:43:17 +0300 Subject: [PATCH 167/167] [Doc] on_startup & on_shutdown signals example with aiopg engine --- docs/web.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/web.rst b/docs/web.rst index 76300f29a49..217a8c47f07 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -1025,6 +1025,32 @@ This can be accomplished by subscribing to the app.on_response_prepare.append(on_prepare) +Additionally, the :attr:`~aiohttp.web.Application.on_startup` and +:attr:`~aiohttp.web.Application.on_shutdown` signals can be subscribed to for +application component setup and tear down accordingly. + +The following example will properly initialize and dispose an aiopg connection +engine:: + + from aiopg.sa import create_engine + + async def create_aiopg(app): + app['pg_engine'] = await create_engine( + user='postgre', + database='postgre', + host='localhost', + port=5432, + password='' + ) + + async def dispose_aiopg(app): + app['pg_engine'].close() + await app['pg_engine'].wait_closed() + + app.on_startup.append(create_aiopg) + app.on_shutdown.append(dispose_aiopg) + + Signal handlers should not return a value but may modify incoming mutable parameters.