diff --git a/setup.cfg b/setup.cfg index 5f18b51f75..44ecd45cd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,9 +56,9 @@ install_requires = Jinja2==3.1.4 ldap3==2.9.1 matterhook==0.2 - meilisearch==0.32.0 + meilisearch==0.33.0 numpy==2.0.1; python_version == '3.9' - numpy==2.1.3; python_version >= '3.10' + numpy==2.2.0; python_version >= '3.10' opencv-python==4.10.0.84 OpenTimelineIO==0.17.0 OpenTimelineIO-Plugins==0.17.0 @@ -72,7 +72,7 @@ install_requires = python-slugify==8.0.4 python-socketio==5.11.4 pytz==2024.2 - redis==5.2.0 + redis==5.2.1 requests==2.32.3 rq==2.0.0 slackclient==2.9.4 @@ -100,12 +100,12 @@ test = fakeredis==2.26.1 mixer==7.2.2 pytest-cov==6.0.0 - pytest==8.3.3 + pytest==8.3.4 monitoring = prometheus-flask-exporter==0.23.1 pygelf==0.4.2 - sentry-sdk==2.19.0 + sentry-sdk==2.19.2 lint = autoflake==2.3.1 diff --git a/tests/auth/test_auth_route.py b/tests/auth/test_auth_route.py index bde8bd34f4..dd1d7043b1 100644 --- a/tests/auth/test_auth_route.py +++ b/tests/auth/test_auth_route.py @@ -290,5 +290,5 @@ def test_get_last_login_logs(self): self.log_in("john.did@gmail.com") login_logs = self.get("/data/events/login-logs/last") self.assertEqual(len(login_logs), 4) - login_logs = self.get("/data/events/login-logs/last?page_size=2") + login_logs = self.get("/data/events/login-logs/last?limit=2") self.assertEqual(len(login_logs), 2) diff --git a/tests/events/test_event_routes.py b/tests/events/test_event_routes.py index 4f2f7ac491..531a630457 100644 --- a/tests/events/test_event_routes.py +++ b/tests/events/test_event_routes.py @@ -34,7 +34,7 @@ def test_get_last_events(self): events = self.get("/data/events/last") self.assertEqual(len(events), 4) - events = self.get("/data/events/last?page_size=2") + events = self.get("/data/events/last?limit=2") self.assertEqual(len(events), 2) events = self.get("/data/events/last?before=%s" % before) self.assertEqual(len(events), 2) diff --git a/tests/services/test_events_service.py b/tests/services/test_events_service.py index acb24d11ba..a56beb16d6 100644 --- a/tests/services/test_events_service.py +++ b/tests/services/test_events_service.py @@ -28,7 +28,7 @@ def test_get_last_events(self): ) events = events_service.get_last_events() self.assertEqual(len(events), 3) - events = events_service.get_last_events(page_size=2) + events = events_service.get_last_events(limit=2) self.assertEqual(len(events), 2) date = fields.get_date_object(asset["created_at"], "%Y-%m-%dT%H:%M:%S") events = events_service.get_last_events(before=date) @@ -44,5 +44,5 @@ def test_get_last_login_logs(self): events_service.create_login_log(self.person.id, "127.0.0.1", "web") login_logs = events_service.get_last_login_logs() self.assertEqual(len(login_logs), 4) - login_logs = events_service.get_last_login_logs(page_size=2) + login_logs = events_service.get_last_login_logs(limit=2) self.assertEqual(len(login_logs), 2) diff --git a/zou/app/__init__.py b/zou/app/__init__.py index 062d87d2e5..6b332cf134 100644 --- a/zou/app/__init__.py +++ b/zou/app/__init__.py @@ -134,7 +134,7 @@ def indexer_key_error(error): raise error -if not config.DEBUG: +if config.DEBUG: @app.errorhandler(Exception) def server_error(error): diff --git a/zou/app/blueprints/auth/resources.py b/zou/app/blueprints/auth/resources.py index 6ddd64486d..d2eb80c1e7 100644 --- a/zou/app/blueprints/auth/resources.py +++ b/zou/app/blueprints/auth/resources.py @@ -78,7 +78,9 @@ def get(self): description: Person not found """ person = persons_service.get_current_user(relations=True) - organisation = persons_service.get_organisation() + organisation = persons_service.get_organisation( + sensitive=permissions.has_admin_permissions() + ) return { "authenticated": True, "user": person, @@ -227,15 +229,21 @@ def post(self): "HTTP_X_REAL_IP", request.remote_addr ) + organisation = persons_service.get_organisation( + sensitive=user["role"] != "admin" + ) + + response = jsonify( + { + "user": user, + "organisation": organisation, + "login": True, + "access_token": access_token, + "refresh_token": refresh_token, + } + ) + if is_from_browser(request.user_agent): - organisation = persons_service.get_organisation() - response = jsonify( - { - "user": user, - "organisation": organisation, - "login": True, - } - ) set_access_cookies(response, access_token) set_refresh_cookies(response, refresh_token) events_service.create_login_log(user["id"], ip_address, "web") @@ -243,12 +251,6 @@ def post(self): events_service.create_login_log( user["id"], ip_address, "script" ) - response = { - "login": True, - "user": user, - "access_token": access_token, - "refresh_token": refresh_token, - } current_app.logger.info(f"User {email} is logged in.") return response except WrongUserException: diff --git a/zou/app/blueprints/breakdown/resources.py b/zou/app/blueprints/breakdown/resources.py index 4297eacd2f..3959c2a980 100644 --- a/zou/app/blueprints/breakdown/resources.py +++ b/zou/app/blueprints/breakdown/resources.py @@ -398,14 +398,14 @@ def get(self, project_id): type: string format: Number x-example: 2 - - in: limit - name: page + - in: query + name: limit required: False type: string format: Number x-example: 100 - - in: cursor_created_at - name: page + - in: query + name: cursor_created_at required: False type: string format: Datetime diff --git a/zou/app/blueprints/crud/base.py b/zou/app/blueprints/crud/base.py index a7bffe4362..151d473467 100644 --- a/zou/app/blueprints/crud/base.py +++ b/zou/app/blueprints/crud/base.py @@ -26,7 +26,22 @@ def all_entries(self, query=None, relations=False): if query is None: query = self.model.query - return self.model.serialize_list(query.all(), relations=relations) + return self.serialize_list(query.all(), relations=relations) + + def serialize_list(self, entries, relations=False): + return self.model.serialize_list( + entries, + relations=relations, + ignored_attrs=( + [] + if permissions.has_admin_permissions() + else [ + "chat_token_slack", + "chat_webhook_mattermost", + "chat_token_discord", + ] + ), + ) def paginated_entries(self, query, page, limit=None, relations=False): total = query.count() diff --git a/zou/app/blueprints/crud/comments.py b/zou/app/blueprints/crud/comments.py index 51a24194f2..9ec74847bc 100644 --- a/zou/app/blueprints/crud/comments.py +++ b/zou/app/blueprints/crud/comments.py @@ -22,19 +22,6 @@ class CommentsResource(BaseModelsResource): def __init__(self): BaseModelsResource.__init__(self, Comment) - def check_read_permissions(self, options=None): - if options is not None: - if "project_id" in options: - user_service.check_project_access(options["project_id"]) - if ( - permissions.has_vendor_permissions() - or permissions.has_client_permissions() - ): - raise permissions.PermissionDenied - else: - return True - return permissions.check_admin_permissions() - class CommentResource(BaseModelResource): def __init__(self): diff --git a/zou/app/blueprints/crud/event.py b/zou/app/blueprints/crud/event.py index c7aedda7cf..35346bfe57 100644 --- a/zou/app/blueprints/crud/event.py +++ b/zou/app/blueprints/crud/event.py @@ -11,7 +11,7 @@ def all_entries(self, query=None, relations=False): if query is None: query = self.model.query - return self.model.serialize_list( + return self.serialize_list( query.limit(1000).all(), relations=relations ) diff --git a/zou/app/blueprints/crud/metadata_descriptor.py b/zou/app/blueprints/crud/metadata_descriptor.py index 504f1a37e7..42afbd7f2f 100644 --- a/zou/app/blueprints/crud/metadata_descriptor.py +++ b/zou/app/blueprints/crud/metadata_descriptor.py @@ -27,15 +27,6 @@ def add_project_permission_filter(self, query): ) return query - def all_entries(self, query=None, relations=True): - if query is None: - query = self.model.query - - return [ - metadata_descriptor.serialize(relations=relations) - for metadata_descriptor in query.all() - ] - def check_creation_integrity(self, data): """ Check if the data descriptor has a valid data_type. @@ -46,6 +37,15 @@ def check_creation_integrity(self, data): raise WrongParameterException("Invalid data_type") return True + def all_entries(self, query=None, relations=True): + if query is None: + query = self.model.query + + return [ + metadata_descriptor.serialize(relations=relations) + for metadata_descriptor in query.all() + ] + class MetadataDescriptorResource(BaseModelResource): diff --git a/zou/app/blueprints/crud/organisation.py b/zou/app/blueprints/crud/organisation.py index 645a5e47ff..cf7cbfb39a 100644 --- a/zou/app/blueprints/crud/organisation.py +++ b/zou/app/blueprints/crud/organisation.py @@ -2,6 +2,7 @@ from zou.app.blueprints.crud.base import BaseModelResource, BaseModelsResource from zou.app.services import persons_service +from zou.app.utils.permissions import has_admin_permissions class OrganisationsResource(BaseModelsResource): @@ -24,6 +25,20 @@ def pre_update(self, instance_dict, data): data["hours_by_day"] = float(data["hours_by_day"]) return data + def serialize_instance(self, data, relations=True): + return data.serialize( + relations=relations, + ignored_attrs=( + [] + if has_admin_permissions() + else [ + "chat_token_slack", + "chat_webhook_mattermost", + "chat_token_discord", + ] + ), + ) + def post_update(self, instance_dict, data): - persons_service.clear_oranisation_cache() + persons_service.clear_organisation_cache() return instance_dict diff --git a/zou/app/blueprints/events/resources.py b/zou/app/blueprints/events/resources.py index 8956764b46..54beab5fd2 100644 --- a/zou/app/blueprints/events/resources.py +++ b/zou/app/blueprints/events/resources.py @@ -32,7 +32,7 @@ def get(self): type: boolean default: False - in: query - name: page_size + name: limit type: integer default: 100 x-example: 100 @@ -50,7 +50,7 @@ def get(self): ("after", None, False), ("before", None, False), ("only_files", False, False), - ("page_size", 100, False), + ("limit", 100, False), ("project_id", None, False), ("name", None, False), ], @@ -59,7 +59,7 @@ def get(self): permissions.check_manager_permissions() before = self.parse_date_parameter(args["before"]) after = self.parse_date_parameter(args["after"]) - page_size = args["page_size"] + limit = args["limit"] only_files = args["only_files"] == "true" project_id = args.get("project_id", None) name = args["name"] @@ -71,7 +71,7 @@ def get(self): return events_service.get_last_events( after=after, before=before, - page_size=page_size, + limit=limit, only_files=only_files, project_id=project_id, name=name, @@ -93,18 +93,18 @@ def get(self): format: date x-example: "2022-07-12T00:00:00" - in: query - name: page_size + name: limit type: integer x-example: 100 responses: 200: description: All login logs """ - args = self.get_args(["before", ("page_size", 100)]) + args = self.get_args(["before", ("limit", 100)]) permissions.check_manager_permissions() before = None if args["before"] is not None: before = date_helpers.get_datetime_from_string(args["before"]) - page_size = args["page_size"] - return events_service.get_last_login_logs(before, page_size) + limit = args["limit"] + return events_service.get_last_login_logs(before, limit) diff --git a/zou/app/blueprints/export/csv/assets.py b/zou/app/blueprints/export/csv/assets.py index f5ffff5dad..8021a997ec 100644 --- a/zou/app/blueprints/export/csv/assets.py +++ b/zou/app/blueprints/export/csv/assets.py @@ -65,7 +65,7 @@ def check_permissions(self, project_id): def build_headers(self, metadata_infos, validation_columns): headers = ["Project", "Type", "Name", "Description", "Time Spent"] - metadata_headers = [name for (name, _) in metadata_infos] + metadata_headers = [name for (name, _, _) in metadata_infos] validation_assignations_columns = [] for validation_column in validation_columns: @@ -97,10 +97,15 @@ def build_row(self, result, metadata_infos, validation_columns): for person_id in task["assignees"] ] ) - - for _, field_name in metadata_infos: - result_metadata = result.get("data", {}) or {} - row.append(result_metadata.get(field_name, "")) + result_data = result.get("data", {}) or {} + for _, field_name, data_type in metadata_infos: + result_metadata = result_data.get(field_name, "") + if data_type == "boolean": + row.append( + "true" if result_metadata.lower() == "true" else "false" + ) + else: + row.append(result_metadata) for column in validation_columns: if column in task_map: @@ -153,7 +158,11 @@ def get_metadata_infos(self, project_id): ] columns = [ - (descriptor["name"], descriptor["field_name"]) + ( + descriptor["name"], + descriptor["field_name"], + descriptor["data_type"], + ) for descriptor in descriptors ] diff --git a/zou/app/blueprints/export/csv/edits.py b/zou/app/blueprints/export/csv/edits.py index 660403357e..7208c17c89 100644 --- a/zou/app/blueprints/export/csv/edits.py +++ b/zou/app/blueprints/export/csv/edits.py @@ -61,7 +61,7 @@ def check_permissions(self, project_id): def build_headers(self, metadata_infos, validation_columns): headers = ["Project", "Episode", "Name", "Description", "Time Spent"] - metadata_headers = [name for (name, _) in metadata_infos] + metadata_headers = [name for (name, _, _) in metadata_infos] validation_assignations_columns = [] for validation_column in validation_columns: @@ -94,9 +94,15 @@ def build_row(self, result, metadata_infos, validation_columns): ] ) - for _, field_name in metadata_infos: - result_metadata = result.get("data", {}) or {} - row.append(result_metadata.get(field_name, "")) + result_data = result.get("data", {}) or {} + for _, field_name, data_type in metadata_infos: + result_metadata = result_data.get(field_name, "") + if data_type == "boolean": + row.append( + "true" if result_metadata.lower() == "true" else "false" + ) + else: + row.append(result_metadata) for column in validation_columns: if column in task_map: @@ -149,7 +155,11 @@ def get_metadata_infos(self, project_id): ] columns = [ - (descriptor["name"], descriptor["field_name"]) + ( + descriptor["name"], + descriptor["field_name"], + descriptor["data_type"], + ) for descriptor in descriptors ] diff --git a/zou/app/blueprints/export/csv/shots.py b/zou/app/blueprints/export/csv/shots.py index 43715c4c6e..f6b265b20a 100644 --- a/zou/app/blueprints/export/csv/shots.py +++ b/zou/app/blueprints/export/csv/shots.py @@ -76,7 +76,7 @@ def build_headers(self, metadata_infos, validation_columns): "FPS", ] - metadata_headers = [name for (name, _) in metadata_infos] + metadata_headers = [name for (name, _, _) in metadata_infos] validation_assignations_columns = [] for validation_column in validation_columns: @@ -123,9 +123,15 @@ def build_row(self, result, metadata_infos, validation_columns): for person_id in task["assignees"] ] ) - - for _, field_name in metadata_infos: - row.append(result.get("data", {}).get(field_name, "")) + result_data = result.get("data", {}) or {} + for _, field_name, data_type in metadata_infos: + result_metadata = result_data.get(field_name, "") + if data_type == "boolean": + row.append( + "true" if result_metadata.lower() == "true" else "false" + ) + else: + row.append(result_metadata) for column in validation_columns: if column in task_map: @@ -171,7 +177,11 @@ def get_metadata_infos(self, project_id): ] columns = [ - (descriptor["name"], descriptor["field_name"]) + ( + descriptor["name"], + descriptor["field_name"], + descriptor["data_type"], + ) for descriptor in descriptors ] diff --git a/zou/app/blueprints/index/resources.py b/zou/app/blueprints/index/resources.py index 37eaaaf1ce..7929715f69 100644 --- a/zou/app/blueprints/index/resources.py +++ b/zou/app/blueprints/index/resources.py @@ -297,6 +297,8 @@ def get(self): ), "saml_enabled": config.SAML_ENABLED, "saml_idp_name": config.SAML_IDP_NAME, + "default_locale": config.DEFAULT_LOCALE, + "default_timezone": config.DEFAULT_TIMEZONE, } if config.SENTRY_KITSU_ENABLED: conf["sentry"] = { diff --git a/zou/app/blueprints/news/resources.py b/zou/app/blueprints/news/resources.py index 68148df329..13538b8c75 100644 --- a/zou/app/blueprints/news/resources.py +++ b/zou/app/blueprints/news/resources.py @@ -21,7 +21,7 @@ def get_news(self, project_ids=[]): episode_id, person_id, page, - page_size, + limit, after, before, ) = self.get_arguments() @@ -38,7 +38,7 @@ def get_news(self, project_ids=[]): episode_id=episode_id, author_id=person_id, page=page, - page_size=page_size, + limit=limit, after=after, before=before, current_user=current_user, @@ -72,7 +72,7 @@ def get_arguments(self): "project_id", "episode_id", {"name": "page", "default": 1, "type": int}, - {"name": "page_size", "default": 50, "type": int}, + {"name": "limit", "default": 50, "type": int}, "after", "before", ], @@ -84,7 +84,7 @@ def get_arguments(self): args["episode_id"], args["person_id"], args["page"], - args["page_size"], + args["limit"], args["after"], args["before"], ) @@ -120,7 +120,7 @@ def get(self, project_id): type: integer x-example: 1 - in: query - name: page_size + name: limit type: integer x-example: 50 - in: query @@ -183,7 +183,7 @@ def get(self): type: integer x-example: 1 - in: query - name: page_size + name: limit type: integer x-example: 50 - in: query diff --git a/zou/app/blueprints/source/csv/assets.py b/zou/app/blueprints/source/csv/assets.py index a5a18515c4..fcc58e36c2 100644 --- a/zou/app/blueprints/source/csv/assets.py +++ b/zou/app/blueprints/source/csv/assets.py @@ -16,7 +16,7 @@ ) from zou.app.models.entity import Entity from zou.app.services.exception import WrongParameterException -from zou.app.utils import events, cache +from zou.app.utils import events, cache, string class AssetsCsvImportResource(BaseCsvProjectImportResource): @@ -209,9 +209,16 @@ def import_row(self, row, project_id): else: asset_new_values["data"] = entity.data.copy() - for name, field_name in self.descriptor_fields.items(): + for name, descriptor in self.descriptor_fields.items(): if name in row: - asset_new_values["data"][field_name] = row[name] + if descriptor["data_type"] == "boolean": + asset_new_values["data"][descriptor["field_name"]] = ( + "true" if string.strtobool(row[name]) else "false" + ) + else: + asset_new_values["data"][descriptor["field_name"]] = row[ + name + ] ready_for = row.get("Ready for", None) if ready_for is not None: diff --git a/zou/app/blueprints/source/csv/base.py b/zou/app/blueprints/source/csv/base.py index 4a9aaa698d..bab260aaaa 100644 --- a/zou/app/blueprints/source/csv/base.py +++ b/zou/app/blueprints/source/csv/base.py @@ -131,5 +131,5 @@ def get_descriptor_field_map(self, project_id, entity_type): descriptors = projects_service.get_metadata_descriptors(project_id) for descriptor in descriptors: if descriptor["entity_type"] == entity_type: - descriptor_map[descriptor["name"]] = descriptor["field_name"] + descriptor_map[descriptor["name"]] = descriptor return descriptor_map diff --git a/zou/app/blueprints/source/csv/edits.py b/zou/app/blueprints/source/csv/edits.py index 16065e669a..1199146761 100644 --- a/zou/app/blueprints/source/csv/edits.py +++ b/zou/app/blueprints/source/csv/edits.py @@ -21,7 +21,7 @@ ) from zou.app.services.comments_service import create_comment from zou.app.services.exception import WrongParameterException -from zou.app.utils import events +from zou.app.utils import events, string class EditsCsvImportResource(BaseCsvProjectImportResource): @@ -192,9 +192,16 @@ def import_row(self, row, project_id): else: edit_new_values["data"] = entity.data.copy() - for name, field_name in self.descriptor_fields.items(): + for name, descriptor in self.descriptor_fields.items(): if name in row: - edit_new_values["data"][field_name] = row[name] + if descriptor["data_type"] == "boolean": + edit_new_values["data"][descriptor["field_name"]] = ( + "true" if string.strtobool(row[name]) else "false" + ) + else: + edit_new_values["data"][descriptor["field_name"]] = row[ + name + ] tasks_update = self.get_tasks_update(row) diff --git a/zou/app/blueprints/source/csv/shots.py b/zou/app/blueprints/source/csv/shots.py index c3c8e0e24d..0fe870d602 100644 --- a/zou/app/blueprints/source/csv/shots.py +++ b/zou/app/blueprints/source/csv/shots.py @@ -21,7 +21,7 @@ ) from zou.app.services.comments_service import create_comment from zou.app.services.exception import WrongParameterException -from zou.app.utils import events +from zou.app.utils import events, string class ShotsCsvImportResource(BaseCsvProjectImportResource): @@ -229,9 +229,16 @@ def import_row(self, row, project_id): if fps is not None: shot_new_values["data"]["fps"] = fps - for name, field_name in self.descriptor_fields.items(): + for name, descriptor in self.descriptor_fields.items(): if name in row: - shot_new_values["data"][field_name] = row[name] + if descriptor["data_type"] == "boolean": + shot_new_values["data"][descriptor["field_name"]] = ( + "true" if string.strtobool(row[name]) else "false" + ) + else: + shot_new_values["data"][descriptor["field_name"]] = row[ + name + ] tasks_update = self.get_tasks_update(row) diff --git a/zou/app/blueprints/tasks/resources.py b/zou/app/blueprints/tasks/resources.py index df88b97604..110484d01c 100644 --- a/zou/app/blueprints/tasks/resources.py +++ b/zou/app/blueprints/tasks/resources.py @@ -1532,6 +1532,11 @@ def get(self, project_id): type: string format: UUID x-example: a24a6ea4-ce75-4665-a070-57453082c25 + - in: query + name: limit + type: integer + default: 100 + x-example: 100 responses: 200: description: All comments to tasks related to given project @@ -1544,7 +1549,8 @@ def get(self, project_id): ): raise permissions.PermissionDenied page = self.get_page() - return tasks_service.get_comments_for_project(project_id, page) + limit = self.get_limit() + return tasks_service.get_comments_for_project(project_id, page, limit) class ProjectPreviewFilesResource(Resource, ArgsMixin): diff --git a/zou/app/models/organisation.py b/zou/app/models/organisation.py index 65b2171c21..06128cda7c 100644 --- a/zou/app/models/organisation.py +++ b/zou/app/models/organisation.py @@ -1,7 +1,6 @@ from zou.app import db from zou.app.models.serializer import SerializerMixin from zou.app.models.base import BaseMixin -from zou.app.utils import fields class Organisation(db.Model, BaseMixin, SerializerMixin): @@ -22,22 +21,15 @@ class Organisation(db.Model, BaseMixin, SerializerMixin): dark_theme_by_default = db.Column(db.Boolean(), default=False) format_duration_in_hours = db.Column(db.Boolean(), default=False) - def present(self): - return fields.serialize_dict( - { - "id": self.id, - "chat_token_slack": self.chat_token_slack, - "chat_webhook_mattermost": self.chat_webhook_mattermost, - "chat_token_discord": self.chat_token_discord, - "name": self.name, - "has_avatar": self.has_avatar, - "hours_by_day": self.hours_by_day, - "hd_by_default": self.hd_by_default, - "use_original_file_name": self.use_original_file_name, - "timesheets_locked": self.timesheets_locked, - "dark_theme_by_default": self.dark_theme_by_default, - "format_duration_in_hours": self.format_duration_in_hours, - "updated_at": self.updated_at, - "created_at": self.created_at, - } + def present(self, sensitive=False): + return self.serialize( + ignored_attrs=( + [] + if sensitive + else [ + "chat_token_slack", + "chat_webhook_mattermost", + "chat_token_discord", + ] + ) ) diff --git a/zou/app/models/serializer.py b/zou/app/models/serializer.py index dd1d802387..3a594af703 100644 --- a/zou/app/models/serializer.py +++ b/zou/app/models/serializer.py @@ -14,35 +14,35 @@ def is_join(self, attr): orm.attributes.CollectionAttributeImpl, ) - def serialize(self, obj_type=None, relations=False, milliseconds=False): + def serialize( + self, + obj_type=None, + relations=False, + milliseconds=False, + ignored_attrs=[], + ): attrs = inspect(self.__class__).all_orm_descriptors.keys() - if relations: - obj_dict = { - attr: serialize_value( - getattr(self, attr), milliseconds=milliseconds - ) - for attr in attrs - } - else: - obj_dict = { - attr: serialize_value( - getattr(self, attr), milliseconds=milliseconds - ) - for attr in attrs - if not self.is_join(attr) - } + obj_dict = { + attr: serialize_value( + getattr(self, attr), milliseconds=milliseconds + ) + for attr in attrs + if attr not in ignored_attrs + and (relations or not self.is_join(attr)) + } obj_dict["type"] = obj_type or type(self).__name__ return obj_dict @staticmethod def serialize_list( - models, obj_type=None, relations=False, milliseconds=False + models, obj_type=None, relations=False, milliseconds=False, **kwargs ): return [ model.serialize( obj_type=obj_type, relations=relations, milliseconds=milliseconds, + **kwargs ) for model in models ] diff --git a/zou/app/services/events_service.py b/zou/app/services/events_service.py index 5c5f91253b..14acedf679 100644 --- a/zou/app/services/events_service.py +++ b/zou/app/services/events_service.py @@ -7,7 +7,7 @@ def get_last_events( after=None, before=None, - page_size=100, + limit=100, only_files=False, project_id=None, name=None, @@ -46,7 +46,7 @@ def get_last_events( if name is not None: query = query.filter(ApiEvent.name == name) - events = query.limit(page_size).all() + events = query.limit(limit).all() return [ fields.serialize_dict( { @@ -71,7 +71,7 @@ def create_login_log(person_id, ip_address, origin): return login_log.serialize() -def get_last_login_logs(before=None, page_size=100): +def get_last_login_logs(before=None, limit=100): """ Return last 100 login logs published. If before parameter is set, it returns last 100 login logs before this date. @@ -85,7 +85,7 @@ def get_last_login_logs(before=None, page_size=100): LoginLog.created_at < func.cast(before, LoginLog.created_at.type) ) - login_logs = query.limit(page_size).all() + login_logs = query.limit(limit).all() return [ { "created_at": fields.serialize_value(created_at), diff --git a/zou/app/services/news_service.py b/zou/app/services/news_service.py index 833cfef89a..8aff9b63a7 100644 --- a/zou/app/services/news_service.py +++ b/zou/app/services/news_service.py @@ -92,7 +92,7 @@ def get_last_news_for_project( task_status_id=None, author_id=None, page=1, - page_size=50, + limit=50, before=None, after=None, episode_id=None, @@ -102,7 +102,7 @@ def get_last_news_for_project( Return last 50 news for given project. Add related information to make it displayable. """ - offset = (page - 1) * page_size + offset = (page - 1) * limit query = ( News.query.order_by(News.created_at.desc()) @@ -158,7 +158,7 @@ def get_last_news_for_project( News.created_at < func.cast(before, News.created_at.type) ) - (total, nb_pages) = _get_news_total(query, page_size) + (total, nb_pages) = _get_news_total(query, limit) query = query.add_columns( Project.id, @@ -172,7 +172,7 @@ def get_last_news_for_project( Entity.preview_file_id, ) - query = query.limit(page_size) + query = query.limit(limit) query = query.offset(offset) news_list = query.all() result = [] @@ -221,15 +221,15 @@ def get_last_news_for_project( "data": result, "total": total, "nb_pages": nb_pages, - "limit": page_size, + "limit": limit, "offset": offset, "page": page, } -def _get_news_total(query, page_size): +def _get_news_total(query, limit): total = query.count() - nb_pages = int(math.ceil(total / float(page_size))) + nb_pages = int(math.ceil(total / float(limit))) return total, nb_pages @@ -314,4 +314,4 @@ def get_news_for_entity(entity_id): """ Get all news related to a given entity. """ - return get_last_news_for_project(entity_id=entity_id, page_size=2000) + return get_last_news_for_project(entity_id=entity_id, limit=2000) diff --git a/zou/app/services/persons_service.py b/zou/app/services/persons_service.py index bdea4e85c2..e75448d1c1 100644 --- a/zou/app/services/persons_service.py +++ b/zou/app/services/persons_service.py @@ -36,8 +36,9 @@ def clear_person_cache(): cache.cache.delete_memoized(get_persons) -def clear_oranisation_cache(): +def clear_organisation_cache(): cache.cache.delete_memoized(get_organisation) + cache.cache.delete_memoized(get_organisation, True) @cache.memoize_function(120) @@ -484,14 +485,14 @@ def invite_person(person_id): @cache.memoize_function(120) -def get_organisation(): +def get_organisation(sensitive=False): """ Return organisation set up on this instance. It creates it if none exists. """ organisation = Organisation.query.first() if organisation is None: organisation = Organisation.create(name="Kitsu") - return organisation.present() + return organisation.present(sensitive=sensitive) def update_organisation(organisation_id, data): @@ -501,7 +502,7 @@ def update_organisation(organisation_id, data): organisation = Organisation.get(organisation_id) organisation.update(data) events.emit("organisation:update", {"organisation_id": organisation_id}) - clear_oranisation_cache() + clear_organisation_cache() return organisation.present() diff --git a/zou/app/services/sync_service.py b/zou/app/services/sync_service.py index 3383e158e6..81e6191bfd 100644 --- a/zou/app/services/sync_service.py +++ b/zou/app/services/sync_service.py @@ -323,12 +323,12 @@ def run_other_sync(project=None, with_events=False): sync_entries("events", ApiEvent, project=project) -def run_last_events_sync(minutes=0, page_size=300): +def run_last_events_sync(minutes=0, limit=300): """ Retrieve last events from source instance and import related data and action. """ - path = "events/last?page_size=%s" % page_size + path = "events/last?limit=%s" % limit if minutes > 0: now = date_helpers.get_utc_now_datetime() min_before = now - datetime.timedelta(minutes=minutes) @@ -346,12 +346,12 @@ def run_last_events_sync(minutes=0, page_size=300): pass -def run_last_events_files(minutes=0, page_size=50): +def run_last_events_files(minutes=0, limit=50): """ Retrieve last events from source instance and import related data and action. """ - path = "events/last?only_files=true&page_size=%s" % page_size + path = "events/last?only_files=true&limit=%s" % limit if minutes > 0: now = date_helpers.get_utc_now_datetime() min_before = now - datetime.timedelta(minutes=minutes) diff --git a/zou/app/services/tasks_service.py b/zou/app/services/tasks_service.py index e9fbb5a691..f07a04356e 100644 --- a/zou/app/services/tasks_service.py +++ b/zou/app/services/tasks_service.py @@ -1659,7 +1659,7 @@ def update_preview_file_info(preview_file): return entity -def get_comments_for_project(project_id, page=0): +def get_comments_for_project(project_id, page=0, limit=None): """ Return all comments for given project. """ @@ -1668,7 +1668,9 @@ def get_comments_for_project(project_id, page=0): .filter(Task.project_id == project_id) .order_by(Comment.updated_at.desc()) ) - return query_utils.get_paginated_results(query, page, relations=True) + return query_utils.get_paginated_results( + query, page, limit, relations=True + ) def get_time_spents_for_project(project_id, page=0): diff --git a/zou/app/services/user_service.py b/zou/app/services/user_service.py index 55a049143a..eca5837c56 100644 --- a/zou/app/services/user_service.py +++ b/zou/app/services/user_service.py @@ -1559,40 +1559,25 @@ def get_timezone(): def get_context(): - if permissions.has_admin_permissions(): - projects = projects_service.open_projects() - else: - projects = get_open_projects() - - asset_types = assets_service.get_asset_types() - custom_actions = custom_actions_service.get_custom_actions() - status_automations = status_automations_service.get_status_automations() - persons = persons_service.get_persons( - minimal=not permissions.has_manager_permissions() - ) - notification_count = get_unread_notifications_count() - project_status_list = projects_service.get_project_statuses() - departments = tasks_service.get_departments() - studios = tasks_service.get_studios() - task_types = tasks_service.get_task_types() - task_status_list = tasks_service.get_task_statuses() - search_filters = get_filters() - search_filter_groups = get_filter_groups() - preview_background_files = files_service.get_preview_background_files() - - return { - "asset_types": asset_types, - "custom_actions": custom_actions, - "status_automations": status_automations, - "departments": departments, - "studios": studios, - "notification_count": notification_count, - "persons": persons, - "project_status": project_status_list, - "projects": projects, - "task_types": task_types, - "task_status": task_status_list, - "search_filters": search_filters, - "search_filter_groups": search_filter_groups, - "preview_background_files": preview_background_files, + context = { + "asset_types": assets_service.get_asset_types(), + "custom_actions": custom_actions_service.get_custom_actions(), + "status_automations": status_automations_service.get_status_automations(), + "departments": tasks_service.get_departments(), + "studios": tasks_service.get_studios(), + "notification_count": get_unread_notifications_count(), + "persons": persons_service.get_persons( + minimal=not permissions.has_manager_permissions() + ), + "project_status": projects_service.get_project_statuses(), + "projects": get_open_projects(), + "task_types": tasks_service.get_task_types(), + "task_status": tasks_service.get_task_statuses(), + "search_filters": get_filters(), + "search_filter_groups": get_filter_groups(), + "preview_background_files": files_service.get_preview_background_files(), } + + if permissions.has_admin_permissions(): + context["user_limit"] = config.USER_LIMIT + return context diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index f21ede412d..f8d61f1c97 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -538,7 +538,7 @@ def run_sync_file_change_daemon( def import_last_changes_from_another_instance( - source, login, password, minutes=0, page_size=300 + source, login, password, minutes=0, limit=300 ): """ Retrieve and save all the data related to most recent events from another @@ -547,12 +547,12 @@ def import_last_changes_from_another_instance( with app.app_context(): sync_service.init(source, login, password) print("Last events syncing started.") - sync_service.run_last_events_sync(minutes=minutes, page_size=300) + sync_service.run_last_events_sync(minutes=minutes, limit=300) print("Last events syncing ended.") def import_last_file_changes_from_another_instance( - source, login, password, minutes=20, page_size=50, force=False + source, login, password, minutes=20, limit=50, force=False ): """ Retrieve and save all the data related most to recent file events @@ -562,7 +562,7 @@ def import_last_file_changes_from_another_instance( with app.app_context(): sync_service.init(source, login, password) print("Last files syncing started.") - sync_service.run_last_events_files(minutes=minutes, page_size=50) + sync_service.run_last_events_files(minutes=minutes, limit=50) print("Last files syncing ended.") diff --git a/zou/app/utils/csv_utils.py b/zou/app/utils/csv_utils.py index 5169d62a4e..3dd9d10504 100644 --- a/zou/app/utils/csv_utils.py +++ b/zou/app/utils/csv_utils.py @@ -1,9 +1,6 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO import csv +from io import StringIO from flask import make_response from slugify import slugify diff --git a/zou/cli.py b/zou/cli.py index 44bd653f1c..ff99a6e342 100755 --- a/zou/cli.py +++ b/zou/cli.py @@ -421,7 +421,7 @@ def sync_file_changes(event_source, source, logs_directory): @click.option("--source", default="http://localhost:8080/api") @click.option("--minutes", default=0) @click.option("--page-size", default=300) -def sync_last_events(source, minutes, page_size): +def sync_last_events(source, minutes, limit): """ Retrieve last events that occured on source instance and import data related to them. It expects that credentials to connect to source instance are @@ -430,7 +430,7 @@ def sync_last_events(source, minutes, page_size): login = os.getenv("SYNC_LOGIN") password = os.getenv("SYNC_PASSWORD") commands.import_last_changes_from_another_instance( - source, login, password, minutes=minutes, page_size=page_size + source, login, password, minutes=minutes, limit=limit ) @@ -438,7 +438,7 @@ def sync_last_events(source, minutes, page_size): @click.option("--source", default="http://localhost:8080/api") @click.option("--minutes", default=20) @click.option("--page-size", default=50) -def sync_last_files(source, minutes, page_size): +def sync_last_files(source, minutes, limit): """ Retrieve last preview files and thumbnails uploaded on source instance. It expects that credentials to connect to source instance are @@ -447,7 +447,7 @@ def sync_last_files(source, minutes, page_size): login = os.getenv("SYNC_LOGIN") password = os.getenv("SYNC_PASSWORD") commands.import_last_file_changes_from_another_instance( - source, login, password, minutes=minutes, page_size=page_size + source, login, password, minutes=minutes, limit=limit ) diff --git a/zou/remote/config_payload.py b/zou/remote/config_payload.py index 804cc675b6..9201e64fbb 100644 --- a/zou/remote/config_payload.py +++ b/zou/remote/config_payload.py @@ -67,7 +67,8 @@ def get_file_from_storage(storage, output_file_path, filename): or os.path.getsize(output_file_path) == 0 ): with open(output_file_path, "wb") as output_file: - output_file.write(storage.read(filename)) + for chunk in storage.read_chunks(filename): + output_file.write(chunk) return output_file_path