Implement Brotli support (#2312)

* Implement Brotli in HttpResponseParserPy

* Add Brotli support to HttpResponseParserC

* fix existed tests

* add tests for brotli support

* add myself to contributors

* write about brotli in docs

* add new fragment into the changes

* fix flake

* fix code coverage

CONTRIBUTORS.txt

@@ -10,6 +10,7 @@ Aleksey Kutepov
 Alex Hayes
 Alex Key
 Alex Khomchenko
+Alex Kuzmenko
 Alex Lisovoy
 Alexander Bayandin
 Alexander Karpinsky

aiohttp/_http_parser.pyx

@@ -161,7 +161,7 @@ cdef class HttpParser:
                               ENCODING_ERR='surrogateescape',
                               CONTENT_ENCODING=hdrs.CONTENT_ENCODING,
                               SEC_WEBSOCKET_KEY1=hdrs.SEC_WEBSOCKET_KEY1,
-                              SUPPORTED=('gzip', 'deflate')):
+                              SUPPORTED=('gzip', 'deflate', 'br')):
         self._process_header()
 
         method = cparser.http_method_str(<cparser.http_method> self._cparser.method)

aiohttp/http_parser.py

@@ -17,6 +17,13 @@
 from .streams import EMPTY_PAYLOAD, FlowControlStreamReader
 
 
+try:
+    import brotli
+    HAS_BROTLI = True
+except ImportError:  # pragma: no cover
+    HAS_BROTLI = False
+
+
 __all__ = (
     'HttpParser', 'HttpRequestParser', 'HttpResponseParser',
     'RawRequestMessage', 'RawResponseMessage')
@@ -324,7 +331,7 @@ def parse_headers(self, lines):
         enc = headers.get(hdrs.CONTENT_ENCODING)
         if enc:
             enc = enc.lower()
-            if enc in ('gzip', 'deflate'):
+            if enc in ('gzip', 'deflate', 'br'):
                 encoding = enc
 
         # chunking
@@ -605,23 +612,29 @@ def __init__(self, out, encoding):
         self.encoding = encoding
         self._started_decoding = False
 
-        zlib_mode = (16 + zlib.MAX_WBITS
-                     if encoding == 'gzip' else -zlib.MAX_WBITS)
-
-        self.zlib = zlib.decompressobj(wbits=zlib_mode)
+        if encoding == 'br':
+            if not HAS_BROTLI:  # pragma: no cover
+                raise ContentEncodingError(
+                    'Can not decode content-encoding: brotli (br). '
+                    'Please install `brotlipy`')
+            self.decompressor = brotli.Decompressor()
+        else:
+            zlib_mode = (16 + zlib.MAX_WBITS
+                         if encoding == 'gzip' else -zlib.MAX_WBITS)
+            self.decompressor = zlib.decompressobj(wbits=zlib_mode)
 
     def set_exception(self, exc):
         self.out.set_exception(exc)
 
     def feed_data(self, chunk, size):
         self.size += size
         try:
-            chunk = self.zlib.decompress(chunk)
+            chunk = self.decompressor.decompress(chunk)
         except Exception:
             if not self._started_decoding and self.encoding == 'deflate':
-                self.zlib = zlib.decompressobj()
+                self.decompressor = zlib.decompressobj()
                 try:
-                    chunk = self.zlib.decompress(chunk)
+                    chunk = self.decompressor.decompress(chunk)
                 except Exception:
                     raise ContentEncodingError(
                         'Can not decode content-encoding: %s' % self.encoding)
@@ -634,11 +647,11 @@ def feed_data(self, chunk, size):
             self.out.feed_data(chunk, len(chunk))
 
     def feed_eof(self):
-        chunk = self.zlib.flush()
+        chunk = self.decompressor.flush()
 
         if chunk or self.size > 0:
             self.out.feed_data(chunk, len(chunk))
-            if not self.zlib.eof:
+            if self.encoding != 'br' and not self.decompressor.eof:
                 raise ContentEncodingError('deflate')
 
         self.out.feed_eof()

changes/2270.feature

@@ -0,0 +1 @@
+Support brotli encoding (generic-purpose lossless compression algorithm)

docs/client.rst

@@ -160,6 +160,9 @@ You can also access the response body as bytes, for non-text requests::
 The ``gzip`` and ``deflate`` transfer-encodings are automatically
 decoded for you.
 
+You can enable ``brotli`` transfer-encodings support,
+just install  `brotlipy <https://github.com/python-hyper/brotlipy>`_.
+
 JSON Response Content
 ---------------------
 

requirements/ci.txt

@@ -18,9 +18,10 @@ pytest-mock==1.6.3
 gunicorn==19.7.1
 twine==1.9.1
 yarl==0.13.0
+brotlipy==0.7.0
 -e .
 
 # Using PEP 508 env markers to control dependency on runtimes:
 aiodns==1.1.1; platform_system!="Windows"  # required c-ares will not build on windows
 codecov==2.0.9; platform_system!="Windows"  # We only use it in Travis CI
-uvloop==0.8.1; python_version>="3.5" and platform_system!="Windows"  # MagicStack/uvloop#14
+uvloop==0.8.1; python_version>="3.5" and platform_system!="Windows"  # MagicStack/uvloop#14

tests/test_http_parser.py

@@ -5,6 +5,7 @@
 import zlib
 from unittest import mock
 
+import brotli
 import pytest
 from multidict import CIMultiDict
 from yarl import URL
@@ -286,6 +287,14 @@ def test_compression_gzip(parser):
     assert msg.compression == 'gzip'
 
 
+def test_compression_brotli(parser):
+    text = (b'GET /test HTTP/1.1\r\n'
+            b'content-encoding: br\r\n\r\n')
+    messages, upgrade, tail = parser.feed_data(text)
+    msg = messages[0][0]
+    assert msg.compression == 'br'
+
+
 def test_compression_unknown(parser):
     text = (b'GET /test HTTP/1.1\r\n'
             b'content-encoding: compress\r\n\r\n')
@@ -686,6 +695,15 @@ def test_http_payload_parser_length_zero(self):
         self.assertTrue(p.done)
         self.assertTrue(out.is_eof())
 
+    def test_http_payload_brotli(self):
+        compressed = brotli.compress(b'brotli data')
+        out = aiohttp.FlowControlDataQueue(self.stream)
+        p = HttpPayloadParser(
+            out, length=len(compressed), compression='br')
+        p.feed_data(compressed)
+        self.assertEqual(b'brotli data', b''.join(d for d, _ in out._buffer))
+        self.assertTrue(out.is_eof())
+
 
 class TestDeflateBuffer(unittest.TestCase):
 
@@ -697,8 +715,8 @@ def test_feed_data(self):
         buf = aiohttp.FlowControlDataQueue(self.stream)
         dbuf = DeflateBuffer(buf, 'deflate')
 
-        dbuf.zlib = mock.Mock()
-        dbuf.zlib.decompress.return_value = b'line'
+        dbuf.decompressor = mock.Mock()
+        dbuf.decompressor.decompress.return_value = b'line'
 
         dbuf.feed_data(b'data', 4)
         self.assertEqual([b'line'], list(d for d, _ in buf._buffer))
@@ -708,8 +726,8 @@ def test_feed_data_err(self):
         dbuf = DeflateBuffer(buf, 'deflate')
 
         exc = ValueError()
-        dbuf.zlib = mock.Mock()
-        dbuf.zlib.decompress.side_effect = exc
+        dbuf.decompressor = mock.Mock()
+        dbuf.decompressor.decompress.side_effect = exc
 
         self.assertRaises(
             http_exceptions.ContentEncodingError, dbuf.feed_data, b'data', 4)
@@ -718,8 +736,8 @@ def test_feed_eof(self):
         buf = aiohttp.FlowControlDataQueue(self.stream)
         dbuf = DeflateBuffer(buf, 'deflate')
 
-        dbuf.zlib = mock.Mock()
-        dbuf.zlib.flush.return_value = b'line'
+        dbuf.decompressor = mock.Mock()
+        dbuf.decompressor.flush.return_value = b'line'
 
         dbuf.feed_eof()
         self.assertEqual([b'line'], list(d for d, _ in buf._buffer))
@@ -729,9 +747,9 @@ def test_feed_eof_err(self):
         buf = aiohttp.FlowControlDataQueue(self.stream)
         dbuf = DeflateBuffer(buf, 'deflate')
 
-        dbuf.zlib = mock.Mock()
-        dbuf.zlib.flush.return_value = b'line'
-        dbuf.zlib.eof = False
+        dbuf.decompressor = mock.Mock()
+        dbuf.decompressor.flush.return_value = b'line'
+        dbuf.decompressor.eof = False
 
         self.assertRaises(http_exceptions.ContentEncodingError, dbuf.feed_eof)