Merge branch 'master' into feature/add-signals-for-reqres-chunks

.travis.yml

@@ -7,7 +7,7 @@ python:
 - &mainstream_python 3.6
 - 3.6-dev
 - nightly
-- &pypy3 pypy3.5-5.8.0
+- &pypy3 pypy3.5
 
 install:
 - &upgrade_python_toolset pip install --upgrade pip wheel setuptools

CHANGES.rst

@@ -14,6 +14,35 @@ Changelog
 
 .. towncrier release notes start
 
+3.0.4 (2018-02-26)
+==================
+
+- Fix ``IndexError`` in HTTP request handling by server. (#2752)
+- Fix MultipartWriter.append* no longer returning part/payload. (#2759)
+
+
+3.0.3 (2018-02-25)
+==================
+
+- Relax ``attrs`` dependency to minimal actually supported version
+  17.0.3 The change allows to avoid version conflicts with currently
+  existing test tools.
+
+3.0.2 (2018-02-23)
+==================
+
+Security Fix
+------------
+
+- Prevent Windows absolute URLs in static files.  Paths like
+  ``/static/D:\path`` and ``/static/\\hostname\drive\path`` are
+  forbidden.
+
+3.0.1
+=====
+
+- Technical release for fixing distribution problems.
+
 3.0.0 (2018-02-12)
 ==================
 

CHANGES/2762.feature

@@ -0,0 +1 @@
+Make ``writer.write_headers`` a coroutine.

aiohttp/client.py

@@ -326,7 +326,7 @@ async def _request(self, method, url, *,
                     tcp_nodelay(conn.transport, True)
                     tcp_cork(conn.transport, False)
                     try:
-                        resp = req.send(conn)
+                        resp = await req.send(conn)
                         try:
                             await resp.start(conn, read_until_eof)
                         except BaseException:

aiohttp/client_reqrep.py

@@ -464,7 +464,7 @@ async def write_bytes(self, writer, conn):
         finally:
             self._writer = None
 
-    def send(self, conn):
+    async def send(self, conn):
         # Specify request target:
         # - CONNECT request must send authority form URI
         # - not CONNECT proxy must send absolute form URI
@@ -514,7 +514,7 @@ async def on_chunk_sent(chunk):
         # status + headers
         status_line = '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format(
             self.method, path, self.version)
-        writer.write_headers(status_line, self.headers)
+        await writer.write_headers(status_line, self.headers)
 
         self._writer = self.loop.create_task(self.write_bytes(writer, conn))
 

aiohttp/connector.py

@@ -893,7 +893,7 @@ async def _create_proxy_connection(self, req, traces=None):
             proxy_req.url = req.url
             key = (req.host, req.port, req.ssl)
             conn = Connection(self, key, proto, self._loop)
-            proxy_resp = proxy_req.send(conn)
+            proxy_resp = await proxy_req.send(conn)
             try:
                 resp = await proxy_resp.start(conn, True)
             except BaseException:

aiohttp/http_writer.py

@@ -99,7 +99,7 @@ def write(self, chunk, *, drain=True, LIMIT=64 * 1024):
 
         return noop()
 
-    def write_headers(self, status_line, headers, SEP=': ', END='\r\n'):
+    async def write_headers(self, status_line, headers, SEP=': ', END='\r\n'):
         """Write request/response status and headers."""
         # status + headers
         headers = status_line + ''.join(

aiohttp/multipart.py

@@ -14,7 +14,7 @@
                    CONTENT_TRANSFER_ENCODING, CONTENT_TYPE)
 from .helpers import CHAR, TOKEN, parse_mimetype, reify
 from .http import HttpParser
-from .payload import (BytesPayload, LookupError, Payload, StringPayload,
+from .payload import (JsonPayload, LookupError, Payload, StringPayload,
                       get_payload, payload_type)
 
 
@@ -712,10 +712,10 @@ def append(self, obj, headers=None):
                 obj.headers.update(headers)
             else:
                 obj._headers = headers
-            self.append_payload(obj)
+            return self.append_payload(obj)
         else:
             try:
-                self.append_payload(get_payload(obj, headers=headers))
+                return self.append_payload(get_payload(obj, headers=headers))
             except LookupError:
                 raise TypeError
 
@@ -752,16 +752,14 @@ def append_payload(self, payload):
         ).encode('utf-8') + b'\r\n'
 
         self._parts.append((payload, headers, encoding, te_encoding))
+        return payload
 
     def append_json(self, obj, headers=None):
         """Helper to append JSON part."""
         if headers is None:
             headers = CIMultiDict()
 
-        data = json.dumps(obj).encode('utf-8')
-        self.append_payload(
-            BytesPayload(
-                data, headers=headers, content_type='application/json'))
+        return self.append_payload(JsonPayload(obj, headers=headers))
 
     def append_form(self, obj, headers=None):
         """Helper to append form urlencoded part."""

aiohttp/test_utils.py

@@ -463,7 +463,6 @@ def make_mocked_request(method, path, headers=None, *,
                         version=HttpVersion(1, 1), closing=False,
                         app=None,
                         writer=sentinel,
-                        payload_writer=sentinel,
                         protocol=sentinel,
                         transport=sentinel,
                         payload=sentinel,
@@ -509,22 +508,20 @@ def make_mocked_request(method, path, headers=None, *,
 
     if writer is sentinel:
         writer = mock.Mock()
+        writer.write_headers = make_mocked_coro(None)
+        writer.write = make_mocked_coro(None)
+        writer.write_eof = make_mocked_coro(None)
+        writer.drain = make_mocked_coro(None)
         writer.transport = transport
 
-    if payload_writer is sentinel:
-        payload_writer = mock.Mock()
-        payload_writer.write = make_mocked_coro(None)
-        payload_writer.write_eof = make_mocked_coro(None)
-        payload_writer.drain = make_mocked_coro(None)
-
     protocol.transport = transport
     protocol.writer = writer
 
     if payload is sentinel:
         payload = mock.Mock()
 
     req = Request(message, payload,
-                  protocol, payload_writer, task, loop,
+                  protocol, writer, task, loop,
                   client_max_size=client_max_size)
 
     match_info = UrlMappingMatchInfo(

aiohttp/web_protocol.py

@@ -244,14 +244,16 @@ def data_received(self, data):
                         500, exc))
                 self.close()
             else:
-                for (msg, payload) in messages:
-                    self._request_count += 1
-                    self._messages.append((msg, payload))
+                if messages:
+                    # sometimes the parser returns no messages
+                    for (msg, payload) in messages:
+                        self._request_count += 1
+                        self._messages.append((msg, payload))
 
-                if self._waiter:
-                    self._waiter.set_result(None)
+                    if self._waiter is not None:
+                        self._waiter.set_result(None)
 
-                self._upgraded = upgraded
+                self._upgrade = upgraded
                 if upgraded and tail:
                     self._message_tail = tail
 

aiohttp/web_response.py

@@ -297,19 +297,19 @@ async def prepare(self, request):
             return self._payload_writer
 
         await request._prepare_hook(self)
-        return self._start(request)
-
-    def _start(self, request,
-               HttpVersion10=HttpVersion10,
-               HttpVersion11=HttpVersion11,
-               CONNECTION=hdrs.CONNECTION,
-               DATE=hdrs.DATE,
-               SERVER=hdrs.SERVER,
-               CONTENT_TYPE=hdrs.CONTENT_TYPE,
-               CONTENT_LENGTH=hdrs.CONTENT_LENGTH,
-               SET_COOKIE=hdrs.SET_COOKIE,
-               SERVER_SOFTWARE=SERVER_SOFTWARE,
-               TRANSFER_ENCODING=hdrs.TRANSFER_ENCODING):
+        return await self._start(request)
+
+    async def _start(self, request,
+                     HttpVersion10=HttpVersion10,
+                     HttpVersion11=HttpVersion11,
+                     CONNECTION=hdrs.CONNECTION,
+                     DATE=hdrs.DATE,
+                     SERVER=hdrs.SERVER,
+                     CONTENT_TYPE=hdrs.CONTENT_TYPE,
+                     CONTENT_LENGTH=hdrs.CONTENT_LENGTH,
+                     SET_COOKIE=hdrs.SET_COOKIE,
+                     SERVER_SOFTWARE=SERVER_SOFTWARE,
+                     TRANSFER_ENCODING=hdrs.TRANSFER_ENCODING):
         self._req = request
 
         keep_alive = self._keep_alive
@@ -364,7 +364,7 @@ def _start(self, request,
         # status line
         status_line = 'HTTP/{}.{} {} {}\r\n'.format(
             version[0], version[1], self._status, self._reason)
-        writer.write_headers(status_line, headers)
+        await writer.write_headers(status_line, headers)
 
         return writer
 
@@ -594,15 +594,15 @@ async def write_eof(self):
         else:
             await super().write_eof()
 
-    def _start(self, request):
+    async def _start(self, request):
         if not self._chunked and hdrs.CONTENT_LENGTH not in self._headers:
             if not self._body_payload:
                 if self._body is not None:
                     self._headers[hdrs.CONTENT_LENGTH] = str(len(self._body))
                 else:
                     self._headers[hdrs.CONTENT_LENGTH] = '0'
 
-        return super()._start(request)
+        return await super()._start(request)
 
     def _do_start_compression(self, coding):
         if self._body_payload or self._chunked:

aiohttp/web_urldispatcher.py

@@ -552,14 +552,22 @@ def __iter__(self):
         return iter(self._routes.values())
 
     async def _handle(self, request):
-        filename = request.match_info['filename']
+        rel_url = request.match_info['filename']
         try:
+            filename = Path(rel_url)
+            if filename.anchor:
+                # rel_url is an absolute name like
+                # /static/\\machine_name\c$ or /static/D:\path
+                # where the static dir is totally different
+                raise HTTPForbidden()
             filepath = self._directory.joinpath(filename).resolve()
             if not self._follow_symlinks:
                 filepath.relative_to(self._directory)
         except (ValueError, FileNotFoundError) as error:
             # relatively safe
             raise HTTPNotFound() from error
+        except HTTPForbidden:
+            raise
         except Exception as error:
             # perm error or other kind!
             request.app.logger.exception(error)

docs/client_reference.rst

@@ -901,7 +901,7 @@ TCPConnector
    :param bool force_close: close underlying sockets after
                             connection releasing (optional).
 
-   :param tuple enable_cleanup_closed: Some ssl servers do not properly complete
+   :param bool enable_cleanup_closed: Some ssl servers do not properly complete
       SSL shutdown process, in that case asyncio leaks SSL connections.
       If this parameter is set to True, aiohttp additionally aborts underlining
       transport after 2 seconds. It is off by default.
@@ -1739,15 +1739,15 @@ Connection errors
 
 .. class:: ClientProxyConnectionError
 
-   Derived from :exc:`ClientConnectonError`
+   Derived from :exc:`ClientConnectorError`
 
 .. class:: ServerConnectionError
 
-   Derived from :exc:`ClientConnectonError`
+   Derived from :exc:`ClientConnectionError`
 
 .. class:: ClientSSLError
 
-   Derived from :exc:`ClientConnectonError`
+   Derived from :exc:`ClientConnectorError`
 
 .. class:: ClientConnectorSSLError
 
@@ -1765,7 +1765,7 @@ Connection errors
 
    Server disconnected.
 
-   Derived from :exc:`ServerDisconnectonError`
+   Derived from :exc:`ServerDisconnectionError`
 
    .. attribute:: message
 
@@ -1776,13 +1776,13 @@ Connection errors
 
    Server operation timeout: read timeout, etc.
 
-   Derived from :exc:`ServerConnectonError` and :exc:`asyncio.TimeoutError`
+   Derived from :exc:`ServerConnectionError` and :exc:`asyncio.TimeoutError`
 
 .. class:: ServerFingerprintMismatch
 
    Server fingerprint mismatch.
 
-   Derived from :exc:`ServerConnectonError`
+   Derived from :exc:`ServerConnectionError`
 
 
 Hierarchy of exceptions

docs/multipart.rst

@@ -152,9 +152,9 @@ will include the file's basename::
     part = root.append(open(__file__, 'rb'))
 
 If you want to send a file with a different name, just handle the
-:class:`BodyPartWriter` instance which :meth:`MultipartWriter.append` will
+:class:`Payload` instance which :meth:`MultipartWriter.append` will
 always return and set `Content-Disposition` explicitly by using
-the :meth:`BodyPartWriter.set_content_disposition` helper::
+the :meth:`Payload.set_content_disposition` helper::
 
     part.set_content_disposition('attachment', filename='secret.txt')
 

docs/powered_by.rst

@@ -14,11 +14,20 @@ make a Pull Request!
 * `Skyscanner Hotels <https://www.skyscanner.net/hotels>`_
 * `Ocean S.A. <https://ocean.io/>`_
 * `GNS3 <http://gns3.com>`_
-* `TutorCruncher socket <https://tutorcruncher.com/features/tutorcruncher-socket/>`_
+* `TutorCruncher socket
+  <https://tutorcruncher.com/features/tutorcruncher-socket/>`_
 * `Morpheus messaging microservice <https://github.com/tutorcruncher/morpheus>`_
 * `Eyepea - Custom telephony solutions <http://www.eyepea.eu>`_
 * `ALLOcloud - Telephony in the cloud <https://www.allocloud.com>`_
-* `helpmanual - comprehensive help and man page database <https://helpmanual.io/>`_
-* `bedevere <https://github.com/python/bedevere>`_ - CPython's GitHub bot, helps maintain and identify issues with a CPython pull request.
-* `miss-islington <https://github.com/python/miss-islington>`_ - CPython's GitHub bot, backports and merge CPython's pull requests
-* `noa technologies - Bike-sharing management platform <https://noa.one/>`_ - SSE endpoint, pushes real time updates of bikes location.
+* `helpmanual - comprehensive help and man page database
+  <https://helpmanual.io/>`_
+* `bedevere <https://github.com/python/bedevere>`_ - CPython's GitHub
+  bot, helps maintain and identify issues with a CPython pull request.
+* `miss-islington <https://github.com/python/miss-islington>`_ -
+  CPython's GitHub bot, backports and merge CPython's pull requests
+* `noa technologies - Bike-sharing management platform
+  <https://noa.one/>`_ - SSE endpoint, pushes real time updates of
+  bikes location.
+* `Wargaming: World of Tanks <https://worldoftanks.ru/>`_
+* `Yandex <https://yandex.ru>`_
+* `Rambler <rambler.ru>`_