Be stricter about JSON that is accepted by Sydent (#337)

This disables the JSON extensions which Python supports by default (parsing of
`Infinity` / `-Infinity` and `NaN`). These shouldn't be accepted since they're
not technically valid JSON and other languages won't be able to interpret it
properly.

This isn't expected to have any observable effects, since no
correctly-operating endpoint should be returning this invalid JSON anyway.

changelog.d/337.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where invalid JSON would be accepted over the HTTP interfaces.

sydent/http/httpclient.py

@@ -25,6 +25,7 @@
 from sydent.http.matrixfederationagent import MatrixFederationAgent
 
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -52,7 +53,7 @@ def get_json(self, uri):
         body = yield readBody(response)
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            json_body = json.loads(body.decode("UTF-8"))
+            json_body = json_decoder.decode(body.decode("UTF-8"))
         except Exception as e:
             logger.exception("Error parsing JSON from %s", uri)
             raise

sydent/http/matrixfederationagent.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 from __future__ import absolute_import
 
-import json
 import logging
 import random
 import time
@@ -32,6 +31,7 @@
 from twisted.web.iweb import IAgent
 
 from sydent.http.srvresolver import SrvResolver, pick_server_from_list
+from sydent.util import json_decoder
 from sydent.util.ttlcache import TTLCache
 
 # period to cache .well-known results for by default
@@ -320,7 +320,7 @@ def _do_get_well_known(self, server_name):
             if response.code != 200:
                 raise Exception("Non-200 response %s" % (response.code, ))
 
-            parsed_body = json.loads(body.decode('utf-8'))
+            parsed_body = json_decoder.decode(body.decode('utf-8'))
             logger.info("Response from .well-known: %s", parsed_body)
             if not isinstance(parsed_body, dict):
                 raise Exception("not a dict")
@@ -439,4 +439,4 @@ class _RoutingResult(object):
     The port we should route the TCP connection to (the target of the SRV record, or
     the port from the URL/.well-known, or 8448)
     :type: int
-    """
+    """

sydent/http/servlets/__init__.py

@@ -22,6 +22,7 @@
 from twisted.internet import defer
 from twisted.web import server
 
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +77,7 @@ def get_args(request, args, required=True):
     ):
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            request_args = json.loads(request.content.read().decode("UTF-8"))
+            request_args = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')
 

sydent/http/servlets/lookupservlet.py

@@ -20,12 +20,11 @@
 from sydent.db.threepid_associations import GlobalAssociationStore
 
 import logging
-import json
 import signedjson.sign
 
 from sydent.http.servlets import get_args, jsonwrap, send_cors, MatrixRestError
 from sydent.http.auth import authIfV2
-
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +41,7 @@ def render_GET(self, request):
         Look up an individual threepid.
 
         ** DEPRECATED **
-        
+
         Params: 'medium': the medium of the threepid
                 'address': the address of the threepid
         Returns: A signed association if the threepid has a corresponding mxid, otherwise the empty object.
@@ -63,7 +62,7 @@ def render_GET(self, request):
         if not sgassoc:
             return {}
 
-        sgassoc = json.loads(sgassoc)
+        sgassoc = json_decoder.decode(sgassoc)
         if not self.sydent.server_name in sgassoc['signatures']:
             # We have not yet worked out what the proper trust model should be.
             #

sydent/http/servlets/replication.py

@@ -19,6 +19,7 @@
 from twisted.web.resource import Resource
 from sydent.http.servlets import jsonwrap, MatrixRestError
 from sydent.threepid import threePidAssocFromDict
+from sydent.util import json_decoder
 
 from sydent.util.hash import sha256_and_url_safe_base64
 
@@ -60,7 +61,7 @@ def render_POST(self, request):
 
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            inJson = json.loads(request.content.read().decode("UTF-8"))
+            inJson = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             logger.warn("Peer %s made push connection with malformed JSON", peer.servername)
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')

sydent/http/servlets/threepidunbindservlet.py

@@ -24,6 +24,7 @@
 
 from sydent.http.servlets import dict_to_json_bytes
 from sydent.db.valsession import ThreePidValSessionStore
+from sydent.util import json_decoder
 from sydent.util.stringutils import is_valid_client_secret
 from sydent.validators import (
     IncorrectClientSecretException,
@@ -51,7 +52,7 @@ def _async_render_POST(self, request):
         try:
             try:
                 # json.loads doesn't allow bytes in Python 3.5
-                body = json.loads(request.content.read().decode("UTF-8"))
+                body = json_decoder.decode(request.content.read().decode("UTF-8"))
             except ValueError:
                 request.setResponseCode(400)
                 request.write(dict_to_json_bytes({'errcode': 'M_BAD_JSON', 'error': 'Malformed JSON'}))
@@ -81,7 +82,7 @@ def _async_render_POST(self, request):
             # and "client_secret" fields, they are trying to prove that they
             # were the original author of the bind. We then check that what
             # they supply matches and if it does, allow the unbind.
-            # 
+            #
             # However if these fields are not supplied, we instead check
             # whether the request originated from a homeserver, and if so the
             # same homeserver that originally created the bind. We do this by
@@ -121,7 +122,7 @@ def _async_render_POST(self, request):
                         'error': "This validation session has not yet been completed"
                     }))
                     return
-                
+
                 if s.medium != threepid['medium'] or s.address != threepid['address']:
                     request.setResponseCode(403)
                     request.write(dict_to_json_bytes({

sydent/replication/peer.py

@@ -22,6 +22,7 @@
 from sydent.db.hashing_metadata import HashingMetadataStore
 from sydent.threepid import threePidAssocFromDict
 from sydent.config import ConfigError
+from sydent.util import json_decoder
 from sydent.util.hash import sha256_and_url_safe_base64
 from unpaddedbase64 import decode_base64
 
@@ -241,7 +242,7 @@ def _failedPushBodyRead(self, body, updateDeferred):
         :param updateDeferred: The deferred to call the error callback of.
         :type updateDeferred: twisted.internet.defer.Deferred
         """
-        errObj = json.loads(body.decode("utf8"))
+        errObj = json_decoder.decode(body.decode("utf8"))
         e = RemotePeerError()
         e.errorDict = errObj
         updateDeferred.errback(e)

sydent/util/__init__.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import time
 
-
 def time_msec():
     """
     Get the current time in milliseconds.
@@ -25,3 +25,12 @@ def time_msec():
     :rtype: int
     """
     return int(time.time() * 1000)
+
+
+def _reject_invalid_json(val):
+    """Do not allow Infinity, -Infinity, or NaN values in JSON."""
+    raise ValueError("Invalid JSON value: '%s'" % val)
+
+
+# a custom JSON decoder which will reject Python extensions to JSON.
+json_decoder = json.JSONDecoder(parse_constant=_reject_invalid_json)