Consistently use simplejson to loading and dumping JSON. (getredash#2817

)

* Consistently use simplejson to loading and dumping JSON.

This introduces the new functions redash.utils.json_dumps and redash.utils.json_loads and simplifies the custom encoder setup.

UUIDs are now handled by the default encoder, too.

Fixes getredash#2807.

Use string comparison in parse_boolean instead of the (simple)json module.

bin/release_manager.py

@@ -1,10 +1,10 @@
 from __future__ import print_function
 import os
 import sys
-import json
 import re
 import subprocess
 import requests
+import simplejson
 
 github_token = os.environ['GITHUB_TOKEN']
 auth = (github_token, 'x-oauth-basic')
@@ -17,7 +17,7 @@ def _github_request(method, path, params=None, headers={}):
         url = path
 
     if params is not None:
-        params = json.dumps(params)
+        params = simplejson.dumps(params)
 
     response = requests.request(method, url, data=params, auth=auth)
     return response

migrations/versions/969126bd800f_.py

@@ -6,7 +6,7 @@
 
 """
 from __future__ import print_function
-import json
+import simplejson
 from alembic import op
 import sqlalchemy as sa
 
@@ -26,7 +26,7 @@ def upgrade():
     print("Updating dashboards position data:")
     for dashboard in Dashboard.query:
         print("  Updating dashboard: {}".format(dashboard.id))
-        layout = json.loads(dashboard.layout)
+        layout = simplejson.loads(dashboard.layout)
 
         print("    Building widgets map:")
         widgets = {}
@@ -47,14 +47,14 @@ def upgrade():
                 if widget is None:
                     continue
 
-                options = json.loads(widget.options) or {}
+                options = simplejson.loads(widget.options) or {}
                 options['position'] = {
                     "row": row_index,
                     "col": column_index * column_size,
                     "sizeX": column_size * widget.width
                 }
 
-                widget.options = json.dumps(options)
+                widget.options = simplejson.dumps(options)
 
                 db.session.add(widget)
 

old_migrations/0003_update_data_source_config.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import json
+import simplejson
 import jsonschema
 from jsonschema import ValidationError
 
@@ -15,7 +15,7 @@ def validate_configuration(query_runner_type, configuration_json):
 
     try:
         if isinstance(configuration_json, string_types):
-            configuration = json.loads(configuration_json)
+            configuration = simplejson.loads(configuration_json)
         else:
             configuration = configuration_json
         jsonschema.validate(configuration, query_runner_class.configuration_schema())
@@ -24,6 +24,7 @@ def validate_configuration(query_runner_type, configuration_json):
 
     return True
 
+
 def update(data_source):
     print("[%s] Old options: %s" % (data_source.name, data_source.options))
 
@@ -40,7 +41,7 @@ def update(data_source):
             if k == 'port':
                 configuration[k] = int(v)
 
-        data_source.options = json.dumps(configuration)
+        data_source.options = simplejson.dumps(configuration)
 
     elif data_source.type == 'mysql':
         mapping = {
@@ -55,10 +56,10 @@ def update(data_source):
         for value in values:
             k, v = value.split("=", 1)
             configuration[mapping[k]] = v
-        data_source.options = json.dumps(configuration)
+        data_source.options = simplejson.dumps(configuration)
 
     elif data_source.type == 'graphite':
-        old_config = json.loads(data_source.options)
+        old_config = simplejson.loads(data_source.options)
 
         configuration = {
             "url": old_config["url"]
@@ -70,13 +71,13 @@ def update(data_source):
         if "auth" in old_config:
             configuration['username'], configuration['password'] = old_config["auth"]
 
-        data_source.options = json.dumps(configuration)
+        data_source.options = simplejson.dumps(configuration)
 
     elif data_source.type == 'url':
-        data_source.options = json.dumps({"url": data_source.options})
+        data_source.options = simplejson.dumps({"url": data_source.options})
 
     elif data_source.type == 'script':
-        data_source.options = json.dumps({"path": data_source.options})
+        data_source.options = simplejson.dumps({"path": data_source.options})
 
     elif data_source.type == 'mongo':
         data_source.type = 'mongodb'

old_migrations/0011_migrate_bigquery_to_json.py

@@ -1,5 +1,5 @@
 from base64 import b64encode
-import json
+import simplejson
 from redash.models import DataSource
 
 
@@ -15,23 +15,23 @@ def convert_p12_to_pem(p12file):
     for ds in DataSource.select(DataSource.id, DataSource.type, DataSource.options):
 
         if ds.type == 'bigquery':
-            options = json.loads(ds.options)
+            options = simplejson.loads(ds.options)
 
             if 'jsonKeyFile' in options:
                 continue
 
             new_options = {
                 'projectId': options['projectId'],
-                'jsonKeyFile': b64encode(json.dumps({
+                'jsonKeyFile': b64encode(simplejson.dumps({
                     'client_email': options['serviceAccount'],
                     'private_key': convert_p12_to_pem(options['privateKey'])
                 }))
             }
 
-            ds.options = json.dumps(new_options)
+            ds.options = simplejson.dumps(new_options)
             ds.save(only=ds.dirty_fields)
         elif ds.type == 'google_spreadsheets':
-            options = json.loads(ds.options)
+            options = simplejson.loads(ds.options)
             if 'jsonKeyFile' in options:
                 continue
 
@@ -40,5 +40,5 @@ def convert_p12_to_pem(p12file):
                     'jsonKeyFile': b64encode(f.read())
                 }
 
-            ds.options = json.dumps(new_options)
+            ds.options = simplejson.dumps(new_options)
             ds.save(only=ds.dirty_fields)

old_migrations/0013_update_counter_options.py

@@ -1,11 +1,11 @@
 from __future__ import print_function
-import json
+import simplejson
 from redash import models
 
 if __name__ == '__main__':
     for vis in models.Visualization.select():
         if vis.type == 'COUNTER':
-            options = json.loads(vis.options)
+            options = simplejson.loads(vis.options)
             print("Before: ", options)
             if 'rowNumber' in options and options['rowNumber'] is not None:
                 options['rowNumber'] += 1
@@ -20,5 +20,5 @@
             options['targetRowNumber'] = options['rowNumber']
 
             print("After: ", options)
-            vis.options = json.dumps(options)
+            vis.options = simplejson.dumps(options)
             vis.save()

redash/admin.py

@@ -1,4 +1,3 @@
-import json
 from flask_admin import Admin
 from flask_admin.base import MenuLink
 from flask_admin.contrib.sqla import ModelView
@@ -8,6 +7,7 @@
 
 from redash import models
 from redash.permissions import require_super_admin
+from redash.utils import json_loads
 
 
 class ArrayListField(fields.Field):
@@ -30,7 +30,7 @@ class JSONTextAreaField(fields.TextAreaField):
     def process_formdata(self, valuelist):
         if valuelist:
             try:
-                json.loads(valuelist[0])
+                json_loads(valuelist[0])
             except ValueError:
                 raise ValueError(self.gettext(u'Invalid JSON'))
             self.data = valuelist[0]

redash/cli/__init__.py

@@ -1,8 +1,6 @@
 from __future__ import print_function
-import json
-
-
 import click
+import simplejson
 from flask.cli import FlaskGroup, run_command
 from flask import current_app
 
@@ -44,7 +42,7 @@ def version():
 
 @manager.command()
 def status():
-    print(json.dumps(get_status(), indent=2))
+    print(simplejson.dumps(get_status(), indent=2))
 
 
 @manager.command()

redash/cli/data_sources.py

@@ -1,6 +1,5 @@
 from __future__ import print_function
 from sys import exit
-import json
 
 import click
 from flask.cli import AppGroup
@@ -10,6 +9,7 @@
 from redash import models
 from redash.query_runner import (get_configuration_schema_for_query_runner_type,
                                  query_runners)
+from redash.utils import json_loads
 from redash.utils.configuration import ConfigurationContainer
 
 manager = AppGroup(help="Data sources management commands.")
@@ -129,7 +129,7 @@ def new(name=None, type=None, options=None, organization='default'):
 
         options = ConfigurationContainer(options_obj, schema)
     else:
-        options = ConfigurationContainer(json.loads(options), schema)
+        options = ConfigurationContainer(json_loads(options), schema)
 
     if not options.is_valid():
         print("Error: invalid configuration.")
@@ -198,7 +198,7 @@ def edit(name, new_name=None, options=None, type=None, organization='default'):
         if options is not None:
             schema = get_configuration_schema_for_query_runner_type(
                 data_source.type)
-            options = json.loads(options)
+            options = json_loads(options)
             data_source.options.set_schema(schema)
             data_source.options.update(options)
 

redash/destinations/__init__.py

@@ -1,5 +1,4 @@
 import logging
-import json
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +54,7 @@ def register(destination_class):
     global destinations
     if destination_class.enabled():
         logger.debug("Registering %s (%s) destinations.", destination_class.name(), destination_class.type())
-        destinations[destination_class.type()] = destination_class 
+        destinations[destination_class.type()] = destination_class
     else:
         logger.warning("%s destination enabled but not supported, not registering. Either disable or install missing dependencies.", destination_class.name())
 

redash/destinations/chatwork.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import requests
 

redash/destinations/hipchat.py

@@ -1,9 +1,9 @@
-import json
 import logging
 import requests
 
 from redash.destinations import *
 from redash.models import Alert
+from redash.utils import json_dumps
 
 
 colors = {
@@ -46,7 +46,7 @@ def notify(self, alert, query, user, new_state, app, host, options):
                 'color': colors.get(new_state, 'green')
             }
             headers = {'Content-Type': 'application/json'}
-            response = requests.post(options['url'], data=json.dumps(data), headers=headers)
+            response = requests.post(options['url'], data=json_dumps(data), headers=headers)
 
             if response.status_code != 204:
                 logging.error('Bad status code received from HipChat: %d', response.status_code)

redash/destinations/mattermost.py

@@ -1,8 +1,8 @@
-import json
 import logging
 import requests
 
 from redash.destinations import *
+from redash.utils import json_dumps
 
 
 class Mattermost(BaseDestination):
@@ -46,7 +46,7 @@ def notify(self, alert, query, user, new_state, app, host, options):
         if options.get('channel'): payload['channel'] = options.get('channel')
 
         try:
-            resp = requests.post(options.get('url'), data=json.dumps(payload))
+            resp = requests.post(options.get('url'), data=json_dumps(payload))
             logging.warning(resp.text)
 
             if resp.status_code != 200:

redash/destinations/slack.py

@@ -1,8 +1,8 @@
-import json
 import logging
 import requests
 
 from redash.destinations import *
+from redash.utils import json_dumps
 
 
 class Slack(BaseDestination):
@@ -58,7 +58,7 @@ def notify(self, alert, query, user, new_state, app, host, options):
         else:
             text = alert.name + " went back to normal"
             color = "#27ae60"
-        
+
         payload = {'attachments': [{'text': text, 'color': color, 'fields': fields}]}
 
         if options.get('username'): payload['username'] = options.get('username')
@@ -67,7 +67,7 @@ def notify(self, alert, query, user, new_state, app, host, options):
         if options.get('channel'): payload['channel'] = options.get('channel')
 
         try:
-            resp = requests.post(options.get('url'), data=json.dumps(payload))
+            resp = requests.post(options.get('url'), data=json_dumps(payload))
             logging.warning(resp.text)
             if resp.status_code != 200:
                 logging.error("Slack send ERROR. status_code => {status}".format(status=resp.status_code))

redash/handlers/admin.py

@@ -1,22 +1,22 @@
-import json
-
 from flask import request
 from flask_login import login_required
+
 from redash import models, redis_connection
 from redash.handlers import routes
 from redash.handlers.base import json_response
 from redash.permissions import require_super_admin
 from redash.serializers import QuerySerializer
-from redash.tasks.queries import QueryTaskTracker
 from redash.tasks import record_event
+from redash.tasks.queries import QueryTaskTracker
+from redash.utils import json_loads
 
 
 @routes.route('/api/admin/queries/outdated', methods=['GET'])
 @require_super_admin
 @login_required
 def outdated_queries():
     manager_status = redis_connection.hgetall('redash:status')
-    query_ids = json.loads(manager_status.get('query_ids', '[]'))
+    query_ids = json_loads(manager_status.get('query_ids', '[]'))
     if query_ids:
         outdated_queries = (
             models.Query.query.outerjoin(models.QueryResult)