Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[qa] use orjson instead of json to improve performances #712

Merged
merged 2 commits into from
Sep 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ install_requires =
matterhook==0.2
meilisearch==0.28.3
OpenTimelineIO==0.15.0
orjson==3.9.7
pillow==10.0.1
psutil==5.9.5
psycopg[binary]==3.1.11
Expand Down
2 changes: 1 addition & 1 deletion tests/auth/test_auth_route.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import json
import orjson as json

from tests.base import ApiDBTestCase

Expand Down
2 changes: 1 addition & 1 deletion tests/base.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import datetime
import unittest
import json
import orjson as json
import os
import ntpath

Expand Down
10 changes: 5 additions & 5 deletions tests/misc/test_commands.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import json
import orjson as json
import datetime

from tests.base import ApiDBTestCase
Expand Down Expand Up @@ -34,7 +34,7 @@ def test_clean_auth_tokens_revoked(self):
},
"revoked": False,
}
).encode("utf-8"),
),
)
self.store.add(
"testkey2",
Expand All @@ -45,7 +45,7 @@ def test_clean_auth_tokens_revoked(self):
},
"revoked": True,
}
).encode("utf-8"),
),
)
self.assertEqual(len(self.store.keys()), 2)
commands.clean_auth_tokens()
Expand All @@ -63,7 +63,7 @@ def test_clean_auth_tokens_expired(self):
},
"revoked": False,
}
).encode("utf-8"),
),
)
self.store.add(
"testkey2",
Expand All @@ -74,7 +74,7 @@ def test_clean_auth_tokens_expired(self):
},
"revoked": False,
}
).encode("utf-8"),
),
)

self.assertEqual(len(self.store.keys()), 2)
Expand Down
2 changes: 1 addition & 1 deletion tests/source/csv/test_import_assets.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import os
import json
import orjson as json

from tests.base import ApiDBTestCase
from zou.app import db
Expand Down
2 changes: 1 addition & 1 deletion tests/source/shotgun/base.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import json
import orjson as json

from tests.base import ApiDBTestCase

Expand Down
3 changes: 2 additions & 1 deletion zou/app/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@

from zou.app.utils import cache, fs, logs
from zou.app.utils.sentry import init_sentry
from zou.app.utils.user_agent import ParsedUserAgent
from zou.app.utils.flask import ParsedUserAgent, ORJSONProvider

init_sentry()
app = Flask(__name__)
app.json = ORJSONProvider(app)
app.request_class.user_agent_class = ParsedUserAgent
app.config.from_object(config)

Expand Down
4 changes: 1 addition & 3 deletions zou/app/blueprints/auth/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,7 @@ def wrong_auth_handler(identity_user=None):

@identity_loaded.connect_via(app)
def on_identity_loaded(sender, identity):
if identity.id is not None:
from zou.app.services import persons_service

if isinstance(identity.id, str):
try:
identity.user = persons_service.get_person(identity.id)

Expand Down
2 changes: 1 addition & 1 deletion zou/app/blueprints/comments/resources.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import json
import orjson as json

from flask import abort, request, send_file as flask_send_file
from flask_restful import Resource, reqparse
Expand Down
2 changes: 1 addition & 1 deletion zou/app/blueprints/crud/base.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import math
import json
import orjson as json
import sqlalchemy.orm as orm

from flask import request, abort, current_app
Expand Down
1 change: 0 additions & 1 deletion zou/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from zou.app.utils.env import envtobool, env_with_semicolon_to_list

PROPAGATE_EXCEPTIONS = True
RESTFUL_JSON = {"ensure_ascii": False}
DEBUG = envtobool("DEBUG", False)
DEBUG_PORT = int(os.getenv("DEBUG_PORT", 5000))

Expand Down
2 changes: 1 addition & 1 deletion zou/app/services/file_tree_service.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os
import re
import json
import orjson as json

from collections import OrderedDict
from slugify import slugify
Expand Down
2 changes: 1 addition & 1 deletion zou/app/services/playlists_service.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import base64

import json
import orjson as json
import os
import zlib

Expand Down
4 changes: 2 additions & 2 deletions zou/app/utils/api.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from flask_restful import Api, output_json
from flask_restful import Api
from zou.app.utils.flask import output_json


def configure_api_from_blueprint(blueprint, route_tuples):
Expand All @@ -13,7 +14,6 @@ def configure_api_from_blueprint(blueprint, route_tuples):
api = Api(blueprint, catch_all_404s=True)

api.representations = {
"application/json; charset=utf-8": output_json,
"application/json": output_json,
}

Expand Down
2 changes: 1 addition & 1 deletion zou/app/utils/commands.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# coding: utf-8

import os
import json
import orjson as json
import datetime
import tempfile

Expand Down
4 changes: 0 additions & 4 deletions zou/app/utils/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,6 @@ def serialize_value(value):
return serialize_orm_arrays(value)
elif isinstance(value, bytes):
return value.decode("utf-8")
elif isinstance(value, str):
return value
elif isinstance(value, int):
return value
elif isinstance(value, list):
return serialize_list(value)
elif isinstance(value, Locale):
Expand Down
51 changes: 51 additions & 0 deletions zou/app/utils/flask.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
from ua_parser import user_agent_parser
from werkzeug.user_agent import UserAgent
from werkzeug.utils import cached_property
from flask.json.provider import JSONProvider
from flask import make_response
import orjson

orjson_options = orjson.OPT_NON_STR_KEYS


def output_json(data, code, headers=None):
"""Makes a Flask response with a JSON encoded body"""
dumped = orjson.dumps(data, option=orjson_options)

resp = make_response(dumped, code)
resp.headers.extend(headers or {})
return resp


class ORJSONProvider(JSONProvider):
def __init__(self, *args, **kwargs):
self.options = kwargs
super().__init__(*args, **kwargs)

def loads(self, s, **kwargs):
return orjson.loads(s)

def dumps(self, obj, **kwargs):
return orjson.dumps(obj, option=orjson_options).decode("utf-8")


class ParsedUserAgent(UserAgent):
@cached_property
def _details(self):
return user_agent_parser.Parse(self.string)

@property
def platform(self):
return self._details["os"]["family"]

@property
def browser(self):
return self._details["user_agent"]["family"]

@property
def version(self):
return ".".join(
part
for key in ("major", "minor", "patch")
if (part := self._details["user_agent"][key]) is not None
)
2 changes: 1 addition & 1 deletion zou/app/utils/remote_job.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import nomad
import base64
import json
import orjson as json
import textwrap
import time

Expand Down
25 changes: 0 additions & 25 deletions zou/app/utils/user_agent.py

This file was deleted.

2 changes: 2 additions & 0 deletions zou/event_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from zou.app import config
from zou.app.stores import auth_tokens_store
from zou.app.utils.sentry import init_sentry
from zou.app.utils.flask import ORJSONProvider

server_stats = {"nb_connections": 0}
rooms_data = {}
Expand Down Expand Up @@ -216,6 +217,7 @@ def create_app():
)
init_sentry()
app = Flask(__name__)
app.json = ORJSONProvider(app)
app.config.from_object(config)
set_info_routes(socketio, app)
set_application_routes(socketio, app)
Expand Down
2 changes: 1 addition & 1 deletion zou/remote/config_payload.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import os
import json
import orjson as json
import sys

from pathlib import Path
Expand Down
2 changes: 1 addition & 1 deletion zou/remote/playlist.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import base64
import json
import orjson as json
import logging
import os
import sys
Expand Down