Merge pull request #712 from EvanBldy/master

[qa] use orjson instead of json to improve performances

setup.cfg

@@ -56,6 +56,7 @@ install_requires =
     matterhook==0.2
     meilisearch==0.28.3
     OpenTimelineIO==0.15.0
+    orjson==3.9.7
     pillow==10.0.1
     psutil==5.9.5
     psycopg[binary]==3.1.11

tests/auth/test_auth_route.py

@@ -1,4 +1,4 @@
-import json
+import orjson as json
 
 from tests.base import ApiDBTestCase
 

tests/base.py

@@ -1,6 +1,6 @@
 import datetime
 import unittest
-import json
+import orjson as json
 import os
 import ntpath
 

tests/misc/test_commands.py

@@ -1,4 +1,4 @@
-import json
+import orjson as json
 import datetime
 
 from tests.base import ApiDBTestCase
@@ -34,7 +34,7 @@ def test_clean_auth_tokens_revoked(self):
                     },
                     "revoked": False,
                 }
-            ).encode("utf-8"),
+            ),
         )
         self.store.add(
             "testkey2",
@@ -45,7 +45,7 @@ def test_clean_auth_tokens_revoked(self):
                     },
                     "revoked": True,
                 }
-            ).encode("utf-8"),
+            ),
         )
         self.assertEqual(len(self.store.keys()), 2)
         commands.clean_auth_tokens()
@@ -63,7 +63,7 @@ def test_clean_auth_tokens_expired(self):
                     },
                     "revoked": False,
                 }
-            ).encode("utf-8"),
+            ),
         )
         self.store.add(
             "testkey2",
@@ -74,7 +74,7 @@ def test_clean_auth_tokens_expired(self):
                     },
                     "revoked": False,
                 }
-            ).encode("utf-8"),
+            ),
         )
 
         self.assertEqual(len(self.store.keys()), 2)

tests/source/csv/test_import_assets.py

@@ -1,5 +1,5 @@
 import os
-import json
+import orjson as json
 
 from tests.base import ApiDBTestCase
 from zou.app import db

tests/source/shotgun/base.py

@@ -1,4 +1,4 @@
-import json
+import orjson as json
 
 from tests.base import ApiDBTestCase
 

zou/app/__init__.py

@@ -29,10 +29,11 @@
 
 from zou.app.utils import cache, fs, logs
 from zou.app.utils.sentry import init_sentry
-from zou.app.utils.user_agent import ParsedUserAgent
+from zou.app.utils.flask import ParsedUserAgent, ORJSONProvider
 
 init_sentry()
 app = Flask(__name__)
+app.json = ORJSONProvider(app)
 app.request_class.user_agent_class = ParsedUserAgent
 app.config.from_object(config)
 

zou/app/blueprints/auth/resources.py

@@ -83,9 +83,7 @@ def wrong_auth_handler(identity_user=None):
 
 @identity_loaded.connect_via(app)
 def on_identity_loaded(sender, identity):
-    if identity.id is not None:
-        from zou.app.services import persons_service
-
+    if isinstance(identity.id, str):
         try:
             identity.user = persons_service.get_person(identity.id)
 

zou/app/blueprints/comments/resources.py

@@ -1,4 +1,4 @@
-import json
+import orjson as json
 
 from flask import abort, request, send_file as flask_send_file
 from flask_restful import Resource, reqparse

zou/app/blueprints/crud/base.py

@@ -1,5 +1,5 @@
 import math
-import json
+import orjson as json
 import sqlalchemy.orm as orm
 
 from flask import request, abort, current_app

zou/app/config.py

@@ -6,7 +6,6 @@
 from zou.app.utils.env import envtobool, env_with_semicolon_to_list
 
 PROPAGATE_EXCEPTIONS = True
-RESTFUL_JSON = {"ensure_ascii": False}
 DEBUG = envtobool("DEBUG", False)
 DEBUG_PORT = int(os.getenv("DEBUG_PORT", 5000))
 

zou/app/services/file_tree_service.py

@@ -1,6 +1,6 @@
 import os
 import re
-import json
+import orjson as json
 
 from collections import OrderedDict
 from slugify import slugify

zou/app/services/playlists_service.py

@@ -1,6 +1,6 @@
 import base64
 
-import json
+import orjson as json
 import os
 import zlib
 

zou/app/utils/api.py

@@ -1,4 +1,5 @@
-from flask_restful import Api, output_json
+from flask_restful import Api
+from zou.app.utils.flask import output_json
 
 
 def configure_api_from_blueprint(blueprint, route_tuples):
@@ -13,7 +14,6 @@ def configure_api_from_blueprint(blueprint, route_tuples):
     api = Api(blueprint, catch_all_404s=True)
 
     api.representations = {
-        "application/json; charset=utf-8": output_json,
         "application/json": output_json,
     }
 

zou/app/utils/commands.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 import os
-import json
+import orjson as json
 import datetime
 import tempfile
 

zou/app/utils/fields.py

@@ -26,10 +26,6 @@ def serialize_value(value):
         return serialize_orm_arrays(value)
     elif isinstance(value, bytes):
         return value.decode("utf-8")
-    elif isinstance(value, str):
-        return value
-    elif isinstance(value, int):
-        return value
     elif isinstance(value, list):
         return serialize_list(value)
     elif isinstance(value, Locale):

zou/app/utils/flask.py

@@ -0,0 +1,51 @@
+from ua_parser import user_agent_parser
+from werkzeug.user_agent import UserAgent
+from werkzeug.utils import cached_property
+from flask.json.provider import JSONProvider
+from flask import make_response
+import orjson
+
+orjson_options = orjson.OPT_NON_STR_KEYS
+
+
+def output_json(data, code, headers=None):
+    """Makes a Flask response with a JSON encoded body"""
+    dumped = orjson.dumps(data, option=orjson_options)
+
+    resp = make_response(dumped, code)
+    resp.headers.extend(headers or {})
+    return resp
+
+
+class ORJSONProvider(JSONProvider):
+    def __init__(self, *args, **kwargs):
+        self.options = kwargs
+        super().__init__(*args, **kwargs)
+
+    def loads(self, s, **kwargs):
+        return orjson.loads(s)
+
+    def dumps(self, obj, **kwargs):
+        return orjson.dumps(obj, option=orjson_options).decode("utf-8")
+
+
+class ParsedUserAgent(UserAgent):
+    @cached_property
+    def _details(self):
+        return user_agent_parser.Parse(self.string)
+
+    @property
+    def platform(self):
+        return self._details["os"]["family"]
+
+    @property
+    def browser(self):
+        return self._details["user_agent"]["family"]
+
+    @property
+    def version(self):
+        return ".".join(
+            part
+            for key in ("major", "minor", "patch")
+            if (part := self._details["user_agent"][key]) is not None
+        )

zou/app/utils/remote_job.py

@@ -1,6 +1,6 @@
 import nomad
 import base64
-import json
+import orjson as json
 import textwrap
 import time
 

zou/event_stream.py

@@ -13,6 +13,7 @@
 from zou.app import config
 from zou.app.stores import auth_tokens_store
 from zou.app.utils.sentry import init_sentry
+from zou.app.utils.flask import ORJSONProvider
 
 server_stats = {"nb_connections": 0}
 rooms_data = {}
@@ -216,6 +217,7 @@ def create_app():
     )
     init_sentry()
     app = Flask(__name__)
+    app.json = ORJSONProvider(app)
     app.config.from_object(config)
     set_info_routes(socketio, app)
     set_application_routes(socketio, app)

zou/remote/config_payload.py

@@ -1,5 +1,5 @@
 import os
-import json
+import orjson as json
 import sys
 
 from pathlib import Path

zou/remote/playlist.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import base64
-import json
+import orjson as json
 import logging
 import os
 import sys