Add serializer parameter to structlog.processors.JSONRenderer

.travis.yml

@@ -35,10 +35,8 @@ matrix:
 
 
     # Meta
-    - python: "2.7"
-      env: TOXENV=flake8-py2
     - python: "3.5"
-      env: TOXENV=flake8-py3
+      env: TOXENV=flake8
     - python: "2.7"
       env: TOXENV=manifest
     - python: "2.7"

CHANGELOG.rst

@@ -24,7 +24,7 @@ Deprecations:
 Changes:
 ^^^^^^^^
 
-*none*
+- Add ``serializer`` parameter to :func:`structlog.processors.JSONRenderer` which allows for using different (possibly faster) JSON encoders than the standard library.
 
 
 15.3.0 (2015-09-25)

dev-requirements.txt

@@ -1,8 +1,6 @@
--r docs-requirements.txt
-check-manifest
-freezegun
+-e .
+coverage
+freezegun>=0.2.8
 pretend
-pytest-cov
 pytest
-twine
-wheel
+simplejson

docs-requirements.txt

@@ -1,5 +1,4 @@
 -e .
-releases
 sphinx
 sphinx_rtd_theme
 twisted

docs/performance.rst

@@ -35,3 +35,12 @@ Here are a few hints how to get most out of ``structlog`` in production:
       configure(cache_logger_on_first_use=True)
 
    This has the only drawback is that later calls on :func:`~structlog.configure` don't have any effect on already cached loggers -- that shouldn't matter outside of testing though.
+#. Use a faster JSON serializer than the standard library.
+   Possible alternatives are among others simplejson_, UltraJSON_, or RapidJSON_ (Python 3 only)::
+
+      structlog.processors.JSONRenderer(serializer=rapidjson.dumps)
+
+
+.. _simplejson: https://simplejson.readthedocs.org/
+.. _UltraJSON: https://github.com/esnme/ultrajson/
+.. _RapidJSON: https://pypi.python.org/pypi/python-rapidjson/

src/structlog/processors.py

@@ -112,6 +112,11 @@ class JSONRenderer(object):
     Render the `event_dict` using `json.dumps(event_dict, **json_kw)`.
 
     :param json_kw: Are passed unmodified to `json.dumps()`.
+    :param callable serializer: A :meth:`json.dumps`-compatible callable that
+        will be used to format the string.  This can be used to use alternative
+        JSON encoders like `simplejson
+        <https://pypi.python.org/pypi/simplejson/>`_ or `RapidJSON
+        <https://pypi.python.org/pypi/python-rapidjson/>`_.
 
     >>> from structlog.processors import JSONRenderer
     >>> JSONRenderer(sort_keys=True)(None, None, {'a': 42, 'b': [1, 2, 3]})
@@ -139,31 +144,28 @@ class JSONRenderer(object):
     .. versionchanged:: 0.2.0
         Added support for ``__structlog__`` serialization method.
     """
-    def __init__(self, **dumps_kw):
+    def __init__(self, serializer=json.dumps, **dumps_kw):
         self._dumps_kw = dumps_kw
+        self._dumps = serializer
 
     def __call__(self, logger, name, event_dict):
-        return json.dumps(event_dict, cls=_JSONFallbackEncoder,
-                          **self._dumps_kw)
+        return self._dumps(event_dict, default=_json_fallback_handler,
+                           **self._dumps_kw)
 
 
-class _JSONFallbackEncoder(json.JSONEncoder):
+def _json_fallback_handler(obj):
     """
     Serialize custom datatypes and pass the rest to __structlog__ & repr().
     """
-    def default(self, obj):
-        """
-        Serialize obj with repr(obj) as fallback.
-        """
-        # circular imports :(
-        from structlog.threadlocal import _ThreadLocalDictWrapper
-        if isinstance(obj, _ThreadLocalDictWrapper):
-            return obj._dict
-        else:
-            try:
-                return obj.__structlog__()
-            except AttributeError:
-                return repr(obj)
+    # circular imports :(
+    from structlog.threadlocal import _ThreadLocalDictWrapper
+    if isinstance(obj, _ThreadLocalDictWrapper):
+        return obj._dict
+    else:
+        try:
+            return obj.__structlog__()
+        except AttributeError:
+            return repr(obj)
 
 
 def format_exc_info(logger, name, event_dict):

tests/test_processors.py

@@ -9,6 +9,12 @@
 import sys
 
 import pytest
+import simplejson
+
+try:
+    import rapidjson
+except ImportError:
+    rapidjson = None
 
 from freezegun import freeze_time
 
@@ -22,7 +28,7 @@
     StackInfoRenderer,
     TimeStamper,
     UnicodeEncoder,
-    _JSONFallbackEncoder,
+    _json_fallback_handler,
     format_exc_info,
 )
 from structlog.threadlocal import wrap_dict
@@ -44,12 +50,18 @@ def __repr__(self):
 
 class TestKeyValueRenderer(object):
     def test_sort_keys(self, event_dict):
+        """
+        Keys are sorted if sort_keys is set.
+        """
         assert (
             r"a=<A(\o/)> b=[3, 4] x=7 y='test' z=(1, 2)" ==
             KeyValueRenderer(sort_keys=True)(None, None, event_dict)
         )
 
     def test_order_complete(self, event_dict):
+        """
+        Orders keys according to key_order.
+        """
         assert (
             r"y='test' b=[3, 4] a=<A(\o/)> z=(1, 2) x=7" ==
             KeyValueRenderer(key_order=['y', 'b', 'a', 'z', 'x'])
@@ -80,29 +92,71 @@ def test_order_extra(self, event_dict):
         )
 
     def test_random_order(self, event_dict):
+        """
+        No special ordering doesn't blow up.
+        """
         rv = KeyValueRenderer()(None, None, event_dict)
         assert isinstance(rv, str)
 
 
 class TestJSONRenderer(object):
     def test_renders_json(self, event_dict):
+        """
+        Renders a predictable JSON string.
+        """
         assert (
             r'{"a": "<A(\\o/)>", "b": [3, 4], "x": 7, "y": "test", "z": '
             r'[1, 2]}' ==
             JSONRenderer(sort_keys=True)(None, None, event_dict)
         )
 
     def test_FallbackEncoder_handles_ThreadLocalDictWrapped_dicts(self):
+        """
+        Our fallback handling handles properly ThreadLocalDictWrapper values.
+        """
         s = json.dumps(wrap_dict(dict)({'a': 42}),
-                       cls=_JSONFallbackEncoder)
+                       default=_json_fallback_handler)
         assert '{"a": 42}' == s
 
     def test_FallbackEncoder_falls_back(self):
+        """
+        The fallback handler uses repr if it doesn't know the type.
+        """
         s = json.dumps({'date': datetime.date(1980, 3, 25)},
-                       cls=_JSONFallbackEncoder,)
+                       default=_json_fallback_handler)
 
         assert '{"date": "datetime.date(1980, 3, 25)"}' == s
 
+    def test_serializer(self):
+        """
+        A custom serializer is used if specified.
+        """
+        jr = JSONRenderer(serializer=lambda obj, **kw: {"a": 42})
+        obj = object()
+
+        assert {"a": 42} == jr(None, None, obj)
+
+    def test_simplejson(self, event_dict):
+        """
+        Integration test with simplejson.
+        """
+        jr = JSONRenderer(serializer=simplejson.dumps)
+
+        assert {
+            'a': '<A(\\o/)>', 'b': [3, 4], 'x': 7, 'y': 'test', 'z': [1, 2]
+        } == json.loads(jr(None, None, event_dict))
+
+    @pytest.mark.skipif(rapidjson is None, reason="rapidjson is missing.")
+    def test_rapidjson(self, event_dict):
+        """
+        Integration test with python-rapidjson.
+        """
+        jr = JSONRenderer(serializer=rapidjson.dumps)
+
+        assert {
+            'a': '<A(\\o/)>', 'b': [3, 4], 'x': 7, 'y': 'test', 'z': [1, 2]
+        } == json.loads(jr(None, None, event_dict))
+
 
 class TestTimeStamper(object):
     def test_disallowsNonUTCUNIXTimestamps(self):

tox.ini

@@ -1,33 +1,25 @@
 [tox]
-envlist = coverage-clean, {py26,py27,py33,py34,py35,pypy}-{threads,greenlets}, flake8-{py2,py3}, docs, manifest, coverage-report
+envlist = coverage-clean,{py26,py27,py33,py34,py35,pypy}-{threads,greenlets},flake8,docs,manifest,coverage-report
 
 [testenv]
 deps =
-    coverage
-    freezegun>=0.2.8
+    -rdev-requirements.txt
     greenlets: greenlet
-    pretend
-    pytest
     py26: ordereddict
     py26: twisted<15.5.0
     py27,py33,py34,py35,pypy: twisted
+    py35: python-rapidjson
 
 setenv =
     PYTHONHASHSEED = 0
     threads: TRICKING_TOX_INTO_GENERATING_AN_ENVIRONMENT = 1
-commands =
-    coverage run --parallel -m pytest tests {posargs}
+commands = coverage run --parallel -m pytest {posargs}
 
-[testenv:flake8-py2]
+[testenv:flake8]
+skip_install = true
+basepython = python3.5
 deps = flake8
-commands =
-    flake8 src tests setup.py
-
-[testenv:flake8-py3]
-basepython = py3: python3.5
-deps = flake8
-commands =
-    flake8 src tests setup.py
+commands = flake8 src tests setup.py
 
 [testenv:docs]
 setenv =
@@ -41,10 +33,9 @@ commands =
     sphinx-build -W -b doctest -d {envtmpdir}/doctrees docs docs/_build/html
 
 [testenv:manifest]
-deps =
-    check-manifest
-commands =
-    check-manifest
+skip_install = true
+deps = check-manifest
+commands = check-manifest
 
 [testenv:coverage-clean]
 deps = coverage