Merge branch 'nghttp2' into development

.coveragerc

@@ -3,3 +3,4 @@ omit =
     hyper/compat.py
     hyper/httplib_compat.py
     hyper/ssl_compat.py
+    hyper/http20/hpack_compat.py

.travis.yml

@@ -9,15 +9,17 @@ python:
 env:
   - TEST_RELEASE=false
   - TEST_RELEASE=true
+  - NGHTTP2=true
 
 matrix:
   allow_failures:
-  - env:
-      TEST_RELEASE=true
+    - env: TEST_RELEASE=true
+  exclude:
+    - env: NGHTTP2=true
+      python: pypy
 
 install:
-  - "pip install ."
-  - "pip install -r test_requirements.txt"
+  - ".travis/install.sh"
 script:
   - >
       if [[ "$TEST_RELEASE" == true ]]; then

.travis/install.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [[ "$NGHTTP2" = true ]]; then
+    # GCC 4.6 seems to cause problems, so go straight to 4.8.
+    sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
+    sudo apt-get update
+    sudo apt-get install g++-4.8 libstdc++-4.8-dev
+    export CXX="g++-4.8" CC="gcc-4.8"
+    $CC --version
+
+    # Install nghttp2. Right now I haven't built a PPA for this so we have to
+    # do it from source, which kinda sucks. First, install a ton of
+    # prerequisite packages.
+    sudo apt-get install autoconf automake autotools-dev libtool pkg-config \
+                         zlib1g-dev libcunit1-dev libssl-dev libxml2-dev \
+                         libevent-dev libjansson-dev libjemalloc-dev
+    pip install cython
+
+    # Now, download and install nghttp2's latest version.
+    wget https://github.com/tatsuhiro-t/nghttp2/releases/download/v0.5.0/nghttp2-0.5.0.tar.gz
+    tar -xzvf nghttp2-0.5.0.tar.gz
+    cd nghttp2-0.5.0
+    autoreconf -i
+    automake
+    autoconf
+    ./configure --disable-threads
+    make
+    sudo make install
+
+    # The makefile doesn't install into the active virtualenv. Install again.
+    cd python
+    python setup.py install
+    cd ../..
+
+    # Let's try ldconfig.
+    sudo sh -c 'echo "/usr/local/lib" > /etc/ld.so.conf.d/libnghttp2.conf'
+    sudo ldconfig
+fi
+
+pip install .
+pip install -r test_requirements.txt

HISTORY.rst

@@ -12,6 +12,8 @@ X.X.X (XXXX-XX-XX)
 - Support for PyPy.
 - Support for Sever Push, thanks to @alekstorm. (`Issue #40`_)
 - Use a buffered socket to avoid unnecessary syscalls. (`Issue #56`_)
+- If `nghttp2`_ is present, use its HPACK encoder for improved speed and
+  compression efficiency. (`Issue #60`_)
 
 *Bugfixes*
 
@@ -40,6 +42,8 @@ X.X.X (XXXX-XX-XX)
 .. _Issue #50: https://github.com/Lukasa/hyper/issues/50
 .. _Issue #52: https://github.com/Lukasa/hyper/issues/52
 .. _Issue #56: https://github.com/Lukasa/hyper/issues/56
+.. _Issue #60: https://github.com/Lukasa/hyper/issues/60
+.. _nghttp2: https://nghttp2.org/
 
 0.0.4 (2014-03-08)
 ------------------

docs/source/advanced.rst

@@ -160,3 +160,27 @@ resources as authoritative without performing this check themselves (since
 the server push mechanism is only an optimization, and clients are free to
 issue requests for any pushed resources manually, there is little downside to
 simply ignoring suspicious ones).
+
+Nghttp2
+-------
+
+By default ``hyper`` uses its built-in pure-Python HPACK encoder and decoder.
+These are reasonably efficient, and suitable for most use cases. However, they
+do not produce the best compression ratio possible, and because they're written
+in pure-Python they incur a cost in memory usage above what is strictly
+necessary.
+
+`nghttp2`_ is a HTTP/2 library written in C that includes a HPACK encoder and
+decoder. ``nghttp2``'s encoder produces extremely compressed output, and
+because it is written in C it is also fast and memory efficient. For this
+reason, performance conscious users may prefer to use ``nghttp2``'s HPACK
+implementation instead of ``hyper``'s.
+
+You can do this very easily. If ``nghttp2``'s Python bindings are installed,
+``hyper`` will transparently switch to using ``nghttp2``'s HPACK implementation
+instead of its own. No configuration is required.
+
+Instructions for installing ``nghttp2`` `are available here`_.
+
+.. _nghttp2: https://nghttp2.org/
+.. _are available here: https://nghttp2.org/documentation/package_README.html#requirements

hyper/http20/connection.py

@@ -5,7 +5,7 @@
 
 Objects that build hyper's connection-level HTTP/2 abstraction.
 """
-from .hpack import Encoder, Decoder
+from .hpack_compat import Encoder, Decoder
 from .stream import Stream
 from .tls import wrap_socket
 from .frame import (

hyper/http20/hpack_compat.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+"""
+hyper/http20/hpack_compat
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Provides an abstraction layer over two HPACK implementations.
+
+Hyper has a pure-Python greenfield HPACK implementation that can be used on
+all Python platforms. However, this implementation is both slower and more
+memory-hungry than could be achieved with a C-language version. Additionally,
+nghttp2's HPACK implementation currently achieves better compression ratios
+than hyper's in almost all benchmarks.
+
+For those who care about efficiency and speed in HPACK, hyper allows you to
+use nghttp2's HPACK implementation instead of hyper's. This module detects
+whether the nghttp2 bindings are installed, and if they are it wraps them in
+a hyper-compatible API and uses them instead of its own. If not, it falls back
+to hyper's built-in Python bindings.
+"""
+import logging
+from .hpack import _to_bytes
+
+log = logging.getLogger(__name__)
+
+# Attempt to import nghttp2.
+try:
+    import nghttp2
+    USE_NGHTTP2 = True
+    log.debug("Using nghttp2's HPACK implementation.")
+except ImportError:
+    USE_NGHTTP2 = False
+    log.debug("Using hyper's pure-Python HPACK implementation.")
+
+if USE_NGHTTP2:
+    class Encoder(object):
+        """
+        An HPACK encoder object. This object takes HTTP headers and emits
+        encoded HTTP/2 header blocks.
+        """
+        def __init__(self):
+            self._e = nghttp2.HDDeflater()
+
+        @property
+        def header_table_size(self):
+            """
+            Returns the header table size. For the moment this isn't
+            useful, so we don't use it.
+            """
+            raise NotImplementedError()
+
+        @header_table_size.setter
+        def header_table_size(self, value):
+            log.debug("Setting header table size to %d", value)
+            self._e.change_table_size(value)
+
+        def encode(self, headers, huffman=True):
+            """
+            Encode the headers. The huffman parameter has no effect, it is
+            simply present for compatibility.
+            """
+            log.debug("HPACK encoding %s", headers)
+
+            # Turn the headers into a list of tuples if possible. This is the
+            # natural way to interact with them in HPACK.
+            if isinstance(headers, dict):
+                headers = headers.items()
+
+            # Next, walk across the headers and turn them all into bytestrings.
+            headers = [(_to_bytes(n), _to_bytes(v)) for n, v in headers]
+
+            # Now, let nghttp2 do its thing.
+            header_block = self._e.deflate(headers)
+
+            return header_block
+
+    class Decoder(object):
+        """
+        An HPACK decoder object.
+        """
+        def __init__(self):
+            self._d = nghttp2.HDInflater()
+
+        @property
+        def header_table_size(self):
+            """
+            Returns the header table size. For the moment this isn't
+            useful, so we don't use it.
+            """
+            raise NotImplementedError()
+
+        @header_table_size.setter
+        def header_table_size(self, value):
+            log.debug("Setting header table size to %d", value)
+            self._d.change_table_size(value)
+
+        def decode(self, data):
+            """
+            Takes an HPACK-encoded header block and decodes it into a header
+            set.
+            """
+            log.debug("Decoding %s", data)
+
+            headers = self._d.inflate(data)
+            return [(n.decode('utf-8'), v.decode('utf-8')) for n, v in headers]
+else:
+    # Grab the built-in encoder and decoder.
+    from .hpack import Encoder, Decoder

test/test_hyper.py

@@ -18,6 +18,7 @@
 from hyper.compat import zlib_compressobj
 from hyper.contrib import HTTP20Adapter
 import errno
+import os
 import pytest
 import socket
 import zlib
@@ -1226,7 +1227,6 @@ def test_connections_handle_resizing_header_tables_properly(self):
 
         # Confirm that the setting is stored and the header table shrunk.
         assert c._settings[SettingsFrame.HEADER_TABLE_SIZE] == 1024
-        assert c.encoder.header_table_size == 1024
 
         # Confirm we got a SETTINGS ACK.
         f2 = decode_frame(sock.queue[0])
@@ -1947,6 +1947,14 @@ def test_splitting_repeated_headers(self):
 
         assert expected == split_repeated_headers(test_headers)
 
+    @pytest.mark.skipif(not os.environ.get('NGHTTP2'), reason="No nghttp2")
+    def test_nghttp2_installs_correctly(self):
+        # This test is a debugging tool: if nghttp2 is being tested by Travis,
+        # we need to confirm it imports correctly. Hyper will normally hide the
+        # import failure, so let's discover it here.
+        import nghttp2
+        assert True
+
 
 # Some utility classes for the tests.
 class NullEncoder(object):