diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a142d8345249..63a4b78e7828 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -325,6 +325,8 @@ jobs: if: failure() run: | docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log + docker logs cvat_worker_export > ${{ github.workspace }}/tests/cvat_worker_export_${{ matrix.specs }}.log + docker logs cvat_worker_import > ${{ github.workspace }}/tests/cvat_worker_import_${{ matrix.specs }}.log - name: Uploading "cvat" container logs as an artifact if: failure() diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py index 4773c307122b..de49b3015e4b 100644 --- a/cvat/apps/engine/mixins.py +++ b/cvat/apps/engine/mixins.py @@ -17,7 +17,6 @@ from cvat.apps.engine.models import Location from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.serializers import DataSerializer -from cvat.apps.webhooks.signals import signal_update, signal_create, signal_delete class TusFile: _tus_cache_timeout = 3600 @@ -334,11 +333,6 @@ def deserialize(self, request, import_func): return self.upload_data(request) -class CreateModelMixin(mixins.CreateModelMixin): - def perform_create(self, serializer, **kwargs): - serializer.save(**kwargs) - signal_create.send(self, instance=serializer.instance) - class PartialUpdateModelMixin: """ Update fields of a model instance. @@ -351,26 +345,8 @@ def _update(self, request, *args, **kwargs): return mixins.UpdateModelMixin.update(self, request, *args, **kwargs) def perform_update(self, serializer): - instance = serializer.instance - data = serializer.to_representation(instance) - old_values = { - attr: data[attr] if attr in data else getattr(instance, attr, None) - for attr in self.request.data.keys() - } - mixins.UpdateModelMixin.perform_update(self, serializer=serializer) - if getattr(serializer.instance, '_prefetched_objects_cache', None): - serializer.instance._prefetched_objects_cache = {} - - signal_update.send(self, instance=serializer.instance, old_values=old_values) - def partial_update(self, request, *args, **kwargs): with mock.patch.object(self, 'update', new=self._update, create=True): return mixins.UpdateModelMixin.partial_update(self, request=request, *args, **kwargs) - - -class DestroyModelMixin(mixins.DestroyModelMixin): - def perform_destroy(self, instance): - signal_delete.send(self, instance=instance) - super().perform_destroy(instance) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 64d13643cc49..0abfa0f30c45 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -542,6 +542,13 @@ def create(cls, **kwargs): except IntegrityError: raise InvalidLabel("All label names must be unique") + def get_organization_id(self): + if self.project is not None: + return self.project.organization.id + if self.task is not None: + return self.task.organization.id + return None + class Meta: default_permissions = () constraints = [ diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index c007f5944045..506f20dc4b85 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -70,7 +70,7 @@ av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message, get_rq_job_meta ) from cvat.apps.engine import backup -from cvat.apps.engine.mixins import PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin, DestroyModelMixin, CreateModelMixin +from cvat.apps.engine.mixins import PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin from cvat.apps.engine.location import get_location_configuration, StorageType from . import models, task @@ -81,6 +81,7 @@ from cvat.apps.engine.cache import MediaCache from cvat.apps.events.handlers import handle_annotations_patch + _UPLOAD_PARSER_CLASSES = api_settings.DEFAULT_PARSER_CLASSES + [MultiPartParser] @extend_schema(tags=['server']) @@ -220,7 +221,7 @@ def plugins(request): }) ) class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, - mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, + mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): queryset = models.Project.objects.select_related( @@ -255,8 +256,7 @@ def get_queryset(self): return queryset def perform_create(self, serializer, **kwargs): - super().perform_create( - serializer, + serializer.save( owner=self.request.user, organization=self.request.iam_context['organization'] ) @@ -673,7 +673,7 @@ def __call__(self, request, start, stop, db_data): }) ) class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, - mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, + mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): queryset = Task.objects.select_related( @@ -797,8 +797,7 @@ def perform_update(self, serializer): updated_instance.project.save() def perform_create(self, serializer, **kwargs): - super().perform_create( - serializer, + serializer.save( owner=self.request.user, organization=self.request.iam_context['organization'] ) @@ -1711,7 +1710,7 @@ def preview(self, request, pk): }) ) class IssueViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, - mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, + mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): queryset = Issue.objects.prefetch_related( @@ -1748,7 +1747,7 @@ def get_serializer_class(self): return IssueWriteSerializer def perform_create(self, serializer, **kwargs): - super().perform_create(serializer, owner=self.request.user) + serializer.save(owner=self.request.user) @extend_schema(tags=['comments']) @extend_schema_view( @@ -1781,7 +1780,7 @@ def perform_create(self, serializer, **kwargs): }) ) class CommentViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, - mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, + mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): queryset = Comment.objects.prefetch_related( @@ -1817,7 +1816,7 @@ def get_serializer_class(self): return CommentWriteSerializer def perform_create(self, serializer, **kwargs): - super().perform_create(serializer, owner=self.request.user) + serializer.save(owner=self.request.user) @extend_schema(tags=['labels']) diff --git a/cvat/apps/events/handlers.py b/cvat/apps/events/handlers.py index cafa29c3b615..819e17942ed6 100644 --- a/cvat/apps/events/handlers.py +++ b/cvat/apps/events/handlers.py @@ -13,7 +13,6 @@ from crum import get_current_user, get_current_request from cvat.apps.engine.models import ( - Organization, Project, Task, Job, @@ -34,13 +33,38 @@ LabelSerializer, ) from cvat.apps.engine.models import ShapeType -from cvat.apps.organizations.serializers import OrganizationReadSerializer -from cvat.apps.webhooks.signals import project_id, organization_id +from cvat.apps.organizations.models import Membership, Organization, Invitation +from cvat.apps.organizations.serializers import OrganizationReadSerializer, MembershipReadSerializer, InvitationReadSerializer from cvat.apps.engine.log import vlogger from .event import event_scope, create_event from .cache import get_cache +def project_id(instance): + if isinstance(instance, Project): + return instance.id + + try: + pid = getattr(instance, "project_id", None) + if pid is None: + return instance.get_project_id() + return pid + except Exception: + return None + + +def organization_id(instance): + if isinstance(instance, Organization): + return instance.id + + try: + oid = getattr(instance, "organization_id", None) + if oid is None: + return instance.get_organization_id() + return oid + except Exception: + return None + def task_id(instance): if isinstance(instance, Task): @@ -66,7 +90,7 @@ def job_id(instance): except Exception: return None -def _get_current_user(instance=None): +def get_user(instance=None): # Try to get current user from request user = get_current_user() if user is not None: @@ -85,7 +109,7 @@ def _get_current_user(instance=None): return None -def _get_current_request(instance=None): +def get_request(instance=None): request = get_current_request() if request is not None: return request @@ -108,19 +132,19 @@ def _get_value(obj, key): return None def request_id(instance=None): - request = _get_current_request(instance) + request = get_request(instance) return _get_value(request, "uuid") def user_id(instance=None): - current_user = _get_current_user(instance) + current_user = get_user(instance) return _get_value(current_user, "id") def user_name(instance=None): - current_user = _get_current_user(instance) + current_user = get_user(instance) return _get_value(current_user, "username") def user_email(instance=None): - current_user = _get_current_user(instance) + current_user = get_user(instance) return _get_value(current_user, "email") def organization_slug(instance): @@ -135,13 +159,13 @@ def organization_slug(instance): except Exception: return None -def _get_instance_diff(old_data, data): - ingone_related_fields = ( +def get_instance_diff(old_data, data): + ignore_related_fields = ( "labels", ) diff = {} for prop, value in data.items(): - if prop in ingone_related_fields: + if prop in ignore_related_fields: continue old_value = old_data.get(prop) if old_value != value: @@ -205,7 +229,7 @@ def _get_object_name(instance): return None -def _get_serializer(instance): +def get_serializer(instance): context = { "request": get_current_request() } @@ -229,8 +253,16 @@ def _get_serializer(instance): serializer = CommentReadSerializer(instance=instance, context=context) if isinstance(instance, Label): serializer = LabelSerializer(instance=instance, context=context) + if isinstance(instance, Membership): + serializer = MembershipReadSerializer(instance=instance, context=context) + if isinstance(instance, Invitation): + serializer = InvitationReadSerializer(instance=instance, context=context) + + return serializer - if serializer : +def get_serializer_without_url(instance): + serializer = get_serializer(instance) + if serializer: serializer.fields.pop("url", None) return serializer @@ -254,7 +286,7 @@ def handle_create(scope, instance, **kwargs): uname = user_name(instance) uemail = user_email(instance) - serializer = _get_serializer(instance=instance) + serializer = get_serializer_without_url(instance=instance) try: payload = serializer.data except Exception: @@ -290,9 +322,9 @@ def handle_update(scope, instance, old_instance, **kwargs): uname = user_name(instance) uemail = user_email(instance) - old_serializer = _get_serializer(instance=old_instance) - serializer = _get_serializer(instance=instance) - diff = _get_instance_diff(old_data=old_serializer.data, data=serializer.data) + old_serializer = get_serializer_without_url(instance=old_instance) + serializer = get_serializer_without_url(instance=instance) + diff = get_instance_diff(old_data=old_serializer.data, data=serializer.data) timestamp = str(datetime.now(timezone.utc).timestamp()) for prop, change in diff.items(): diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 7d59481c8c9f..398294712752 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -8,7 +8,7 @@ from django.utils.crypto import get_random_string from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view -from cvat.apps.engine.mixins import PartialUpdateModelMixin, DestroyModelMixin, CreateModelMixin +from cvat.apps.engine.mixins import PartialUpdateModelMixin from cvat.apps.iam.permissions import ( InvitationPermission, MembershipPermission, OrganizationPermission) @@ -112,7 +112,7 @@ class Meta: '204': OpenApiResponse(description='The membership has been deleted'), }) ) -class MembershipViewSet(mixins.RetrieveModelMixin, DestroyModelMixin, +class MembershipViewSet(mixins.RetrieveModelMixin, mixins.DestroyModelMixin, mixins.ListModelMixin, PartialUpdateModelMixin, viewsets.GenericViewSet): queryset = Membership.objects.all() ordering = '-id' @@ -170,8 +170,8 @@ class InvitationViewSet(viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin, PartialUpdateModelMixin, - CreateModelMixin, - DestroyModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, ): queryset = Invitation.objects.all() http_method_names = ['get', 'post', 'patch', 'delete', 'head', 'options'] @@ -196,13 +196,12 @@ def get_queryset(self): permission = InvitationPermission.create_scope_list(self.request) return permission.filter(queryset) - def perform_create(self, serializer, **kwargs): - extra_kwargs = { - 'owner': self.request.user, - 'key': get_random_string(length=64), - 'organization': self.request.iam_context['organization'] - } - super().perform_create(serializer, **extra_kwargs) + def perform_create(self, serializer): + serializer.save( + owner=self.request.user, + key=get_random_string(length=64), + organization=self.request.iam_context['organization'] + ) def perform_update(self, serializer): if 'accepted' in self.request.query_params: diff --git a/cvat/apps/webhooks/event_type.py b/cvat/apps/webhooks/event_type.py index 4c74810f54dc..59cdb6cf99ed 100644 --- a/cvat/apps/webhooks/event_type.py +++ b/cvat/apps/webhooks/event_type.py @@ -13,12 +13,12 @@ class Events: RESOURCES = { "project": ["create", "update", "delete"], "task": ["create", "update", "delete"], + "job": ["create", "update", "delete"], "issue": ["create", "update", "delete"], "comment": ["create", "update", "delete"], - "invitation": ["create", "delete"], # TO-DO: implement invitation_updated, - "membership": ["update", "delete"], - "job": ["update"], - "organization": ["update"], + "organization": ["update", "delete"], + "invitation": ["create", "delete"], + "membership": ["create", "update", "delete"], } @classmethod @@ -47,12 +47,9 @@ class AllEvents: class ProjectEvents: webhook_type = WebhookTypeChoice.PROJECT - events = [event_name("update", "project")] \ - + Events.select(["job", "task", "issue", "comment"]) + events = [*Events.select(["task", "job", "label", "issue", "comment"]), event_name("update", "project"), event_name("delete", "project")] class OrganizationEvents: webhook_type = WebhookTypeChoice.ORGANIZATION - events = [event_name("update", "organization")] \ - + Events.select(["membership", "invitation", "project"]) \ - + ProjectEvents.events + events = AllEvents.events diff --git a/cvat/apps/webhooks/signals.py b/cvat/apps/webhooks/signals.py index 5bc7444ac219..cee8521c4d71 100644 --- a/cvat/apps/webhooks/signals.py +++ b/cvat/apps/webhooks/signals.py @@ -4,17 +4,24 @@ import hashlib import hmac -from http import HTTPStatus import json +from copy import deepcopy +from http import HTTPStatus import django_rq import requests -from django.dispatch import Signal, receiver from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.db.models.signals import (post_delete, post_save, pre_delete, + pre_save) +from django.dispatch import Signal, receiver -from cvat.apps.engine.models import Project +from cvat.apps.engine.models import Comment, Issue, Job, Project, Task from cvat.apps.engine.serializers import BasicUserSerializer -from cvat.apps.organizations.models import Organization +from cvat.apps.events.handlers import (get_request, get_serializer, get_user, + get_instance_diff, organization_id, + project_id) +from cvat.apps.organizations.models import Invitation, Membership, Organization from cvat.utils.http import make_requests_session from .event_type import EventTypeChoice, event_name @@ -23,14 +30,10 @@ WEBHOOK_TIMEOUT = 10 RESPONSE_SIZE_LIMIT = 1 * 1024 * 1024 # 1 MB -signal_update = Signal() -signal_create = Signal() -signal_delete = Signal() signal_redelivery = Signal() signal_ping = Signal() - -def send_webhook(webhook, payload, delivery): +def send_webhook(webhook, payload, redelivery=False): headers = {} if webhook.secret: headers["X-Signature-256"] = ( @@ -54,161 +57,183 @@ def send_webhook(webhook, payload, delivery): stream=True, ) status_code = response.status_code - response_body = response.raw.read(RESPONSE_SIZE_LIMIT + 1, decode_content=True) + response_body = response.raw.read( + RESPONSE_SIZE_LIMIT + 1, decode_content=True + ) except requests.ConnectionError: status_code = HTTPStatus.BAD_GATEWAY except requests.Timeout: status_code = HTTPStatus.GATEWAY_TIMEOUT - setattr(delivery, "status_code", status_code) + response = "" if response_body is not None and len(response_body) < RESPONSE_SIZE_LIMIT + 1: - setattr(delivery, "response", response_body.decode("utf-8")) + response = response_body.decode("utf-8") - delivery.save() - - -def add_to_queue(webhook, payload, redelivery=False): delivery = WebhookDelivery.objects.create( webhook_id=webhook.id, event=payload["event"], - status_code=None, + status_code=status_code, changed_fields=",".join(list(payload.get("before_update", {}).keys())), redelivery=redelivery, request=payload, - response="", + response=response, ) + return delivery + +def add_to_queue(webhook, payload, redelivery=False): queue = django_rq.get_queue(settings.CVAT_QUEUES.WEBHOOKS.value) - queue.enqueue_call(func=send_webhook, args=(webhook, payload, delivery)) + queue.enqueue_call(func=send_webhook, args=(webhook, payload, redelivery)) - return delivery + +def batch_add_to_queue(webhooks, data): + payload = deepcopy(data) + for webhook in webhooks: + payload["webhook_id"] = webhook.id + add_to_queue(webhook, payload) -def select_webhooks(project_id, org_id, event): +def select_webhooks(instance, event): selected_webhooks = [] - if org_id is not None: + pid = project_id(instance) + oid = organization_id(instance) + if oid is not None: webhooks = Webhook.objects.filter( is_active=True, events__contains=event, type=WebhookTypeChoice.ORGANIZATION, - organization=org_id, + organization=oid, ) selected_webhooks += list(webhooks) - if project_id is not None: + if pid is not None: webhooks = Webhook.objects.filter( is_active=True, events__contains=event, type=WebhookTypeChoice.PROJECT, - project=project_id, + project=pid, ) selected_webhooks += list(webhooks) return selected_webhooks -def payload(data, request): - return { - **data, - "sender": BasicUserSerializer(request.user, context={"request": request}).data, - } - - -def project_id(instance): - if isinstance(instance, Project): - return instance.id - - try: - pid = getattr(instance, "project_id", None) - if pid is None: - return instance.get_project_id() - return pid - except Exception: - return None - +def get_sender(instance): + user = get_user(instance) + if isinstance(user, dict): + return user + return BasicUserSerializer(user, context={"request": get_request(instance)}).data -def organization_id(instance): - if isinstance(instance, Organization): - return instance.id +@receiver(pre_save, sender=Project, dispatch_uid=__name__ + ":project:pre_save") +@receiver(pre_save, sender=Task, dispatch_uid=__name__ + ":task:pre_save") +@receiver(pre_save, sender=Job, dispatch_uid=__name__ + ":job:pre_save") +@receiver(pre_save, sender=Issue, dispatch_uid=__name__ + ":issue:pre_save") +@receiver(pre_save, sender=Comment, dispatch_uid=__name__ + ":comment:pre_save") +@receiver(pre_save, sender=Organization, dispatch_uid=__name__ + ":organization:pre_save") +@receiver(pre_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:pre_save") +@receiver(pre_save, sender=Membership, dispatch_uid=__name__ + ":membership:pre_save") +def pre_save_resource_event(sender, instance, **kwargs): try: - oid = getattr(instance, "organization_id", None) - if oid is None: - return instance.get_organization_id() - return oid - except Exception: - return None - - -@receiver(signal_update) -def update(sender, instance=None, old_values=None, **kwargs): - event = event_name("update", sender.basename) - if event not in map(lambda a: a[0], EventTypeChoice.choices()): + old_instance = sender.objects.get(pk=instance.pk) + except ObjectDoesNotExist: return - serializer = sender.get_serializer_class()( - instance=instance, context={"request": sender.request} - ) - - pid = project_id(instance) - oid = organization_id(instance) + old_serializer = get_serializer(instance=old_instance) + serializer = get_serializer(instance=instance) + diff = get_instance_diff(old_data=old_serializer.data, data=serializer.data) - if not any((oid, pid)): + if not diff: return - data = { - "event": event, - sender.basename: serializer.data, - "before_update": old_values, + before_update = { + attr: value["old_value"] + for attr, value in diff.items() } - for webhook in select_webhooks(pid, oid, event): - data.update({"webhook_id": webhook.id}) - add_to_queue(webhook, payload(data, sender.request)) + instance._before_update = before_update -@receiver(signal_create) -def resource_created(sender, instance=None, **kwargs): - event = event_name("create", sender.basename) - if event not in map(lambda a: a[0], EventTypeChoice.choices()): - return +@receiver(post_save, sender=Project, dispatch_uid=__name__ + ":project:post_save") +@receiver(post_save, sender=Task, dispatch_uid=__name__ + ":task:post_save") +@receiver(post_save, sender=Job, dispatch_uid=__name__ + ":job:post_save") +@receiver(post_save, sender=Issue, dispatch_uid=__name__ + ":issue:post_save") +@receiver(post_save, sender=Comment, dispatch_uid=__name__ + ":comment:post_save") +@receiver(post_save, sender=Organization, dispatch_uid=__name__ + ":organization:post_save") +@receiver(post_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:post_save") +@receiver(post_save, sender=Membership, dispatch_uid=__name__ + ":membership:post_save") +def post_save_resource_event(sender, instance, created, **kwargs): + resource_name = instance.__class__.__name__.lower() - pid = project_id(instance) - oid = organization_id(instance) - if not any((oid, pid)): + event_type = event_name("create" if created else "update", resource_name) + if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): return - serializer = sender.get_serializer_class()( - instance=instance, context={"request": sender.request} - ) - - data = {"event": event, sender.basename: serializer.data} - - for webhook in select_webhooks(pid, oid, event): - data.update({"webhook_id": webhook.id}) - add_to_queue(webhook, payload(data, sender.request)) - - -@receiver(signal_delete) -def resource_deleted(sender, instance=None, **kwargs): - event = event_name("delete", sender.basename) - if event not in map(lambda a: a[0], EventTypeChoice.choices()): + filtered_webhooks = select_webhooks(instance, event_type) + if not filtered_webhooks: return - pid = project_id(instance) - oid = organization_id(instance) - if not any((oid, pid)): + data = { + "event": event_type, + resource_name: get_serializer(instance=instance).data, + "sender": get_sender(instance), + } + + if not created: + if before_update := getattr(instance, "_before_update", None): + data["before_update"] = before_update + else: + return + + batch_add_to_queue(filtered_webhooks, data) + + +@receiver(pre_delete, sender=Project, dispatch_uid=__name__ + ":project:pre_delete") +@receiver(pre_delete, sender=Task, dispatch_uid=__name__ + ":task:pre_delete") +@receiver(pre_delete, sender=Job, dispatch_uid=__name__ + ":job:pre_delete") +@receiver(pre_delete, sender=Issue, dispatch_uid=__name__ + ":issue:pre_delete") +@receiver(pre_delete, sender=Comment, dispatch_uid=__name__ + ":comment:pre_delete") +@receiver(pre_delete, sender=Organization, dispatch_uid=__name__ + ":organization:pre_delete") +@receiver(pre_delete, sender=Invitation, dispatch_uid=__name__ + ":invitation:pre_delete") +@receiver(pre_delete, sender=Membership, dispatch_uid=__name__ + ":membership:pre_delete") +def pre_delete_resource_event(sender, instance, **kwargs): + resource_name = instance.__class__.__name__.lower() + + related_webhooks = [] + if resource_name in ["project", "organization"]: + related_webhooks = select_webhooks(instance, event_name("delete", resource_name)) + + serializer = get_serializer(instance=deepcopy(instance)) + instance._deleted_object = dict(serializer.data) + instance._related_webhooks = related_webhooks + + +@receiver(post_delete, sender=Project, dispatch_uid=__name__ + ":project:post_delete") +@receiver(post_delete, sender=Task, dispatch_uid=__name__ + ":task:post_delete") +@receiver(post_delete, sender=Job, dispatch_uid=__name__ + ":job:post_delete") +@receiver(post_delete, sender=Issue, dispatch_uid=__name__ + ":issue:post_delete") +@receiver(post_delete, sender=Comment, dispatch_uid=__name__ + ":comment:post_delete") +@receiver(post_delete, sender=Organization, dispatch_uid=__name__ + ":organization:post_delete") +@receiver(post_delete, sender=Invitation, dispatch_uid=__name__ + ":invitation:post_delete") +@receiver(post_delete, sender=Membership, dispatch_uid=__name__ + ":membership:post_delete") +def post_delete_resource_event(sender, instance, **kwargs): + resource_name = instance.__class__.__name__.lower() + + event_type = event_name("delete", resource_name) + if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): return - serializer = sender.get_serializer_class()( - instance=instance, context={"request": sender.request} - ) + filtered_webhooks = select_webhooks(instance, event_type) - data = {"event": event, sender.basename: serializer.data} + data = { + "event": event_type, + resource_name: getattr(instance, "_deleted_object"), + "sender": get_sender(instance), + } - for webhook in select_webhooks(pid, oid, event): - data.update({"webhook_id": webhook.id}) - add_to_queue(webhook, payload(data, sender.request)) + batch_add_to_queue(filtered_webhooks, data) + related_webhooks = [webhook for webhook in getattr(instance, "_related_webhooks", []) if webhook.id not in map(lambda a: a.id, filtered_webhooks)] + batch_add_to_queue(related_webhooks, data) @receiver(signal_redelivery) @@ -218,6 +243,6 @@ def redelivery(sender, data=None, **kwargs): @receiver(signal_ping) def ping(sender, serializer, **kwargs): - data = {"event": "ping", "webhook": serializer.data} - delivery = add_to_queue(serializer.instance, payload(data, sender.request)) + data = {"event": "ping", "webhook": serializer.data, "sender": get_sender(serializer.instance)} + delivery = send_webhook(serializer.instance, data, redelivery=False) return delivery diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index 86216d018a54..10972e58882d 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -77,10 +77,6 @@ class WebhookViewSet(viewsets.ModelViewSet): iam_organization_field = "organization" def get_serializer_class(self): - # Early exit for drf-spectacular compatibility - if getattr(self, "swagger_fake_view", False): - return WebhookReadSerializer - if self.request.path.endswith("redelivery") or self.request.path.endswith( "ping" ): @@ -185,9 +181,7 @@ def retrieve_delivery(self, request, pk, delivery_id): def redelivery(self, request, pk, delivery_id): delivery = WebhookDelivery.objects.get(webhook_id=pk, id=delivery_id) signal_redelivery.send(sender=self, data=delivery.request) - - # Questionable: should we provide a body for this response? - return Response({}) + return Response({}, status=status.HTTP_200_OK) @extend_schema( summary="Method send ping webhook", diff --git a/cvat/schema.yml b/cvat/schema.yml index 50de2027d7bc..01163eb7b622 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -7213,12 +7213,16 @@ components: - create:comment - create:invitation - create:issue + - create:job + - create:membership - create:project - create:task - delete:comment - delete:invitation - delete:issue + - delete:job - delete:membership + - delete:organization - delete:project - delete:task - update:comment diff --git a/tests/python/rest_api/test_webhooks_sender.py b/tests/python/rest_api/test_webhooks_sender.py index 5b1570c72917..2009931cb13f 100644 --- a/tests/python/rest_api/test_webhooks_sender.py +++ b/tests/python/rest_api/test_webhooks_sender.py @@ -72,10 +72,8 @@ def get_deliveries(webhook_id): assert response.status_code == HTTPStatus.OK deliveries = response.json() - delivery = deliveries["results"][0]["response"] - - if delivery: - delivery_response = json.loads(delivery) + if deliveries["count"] > 0: + delivery_response = json.loads(deliveries["results"][0]["response"]) break sleep(1) @@ -119,27 +117,7 @@ def test_webhook_update_project_name(self): == {} ) - def test_webhook_update_project_labels(self): - response = post_method("admin1", "projects", {"name": "project"}) - assert response.status_code == HTTPStatus.CREATED - project = response.json() - - events = ["update:project"] - webhook = create_webhook(events, "project", project["id"]) - - patch_data = {"labels": [{"name": "label_0", "color": "#aabbcc"}]} - response = patch_method("admin1", f"projects/{project['id']}", patch_data) - assert response.status_code == HTTPStatus.OK - - deliveries, payload = get_deliveries(webhook["id"]) - - assert deliveries["count"] == 1 - - assert payload["event"] == events[0] - assert payload["before_update"]["labels"]["count"] == 0 - assert payload["project"]["labels"]["count"] == 1 - - def test_webhook_create_and_delete_project(self, organizations): + def test_webhook_create_and_delete_project_in_organization(self, organizations): org_id = list(organizations)[0]["id"] events = ["create:project", "delete:project"] @@ -319,33 +297,9 @@ def test_webhook_update_task_assignee(self, users, tasks): deliveries, payload = get_deliveries(webhook_id=webhook_id) assert deliveries["count"] == 1 - assert payload["before_update"]["assignee_id"] == tasks[task_id]["assignee"]["id"] + assert payload["before_update"]["assignee"]["id"] == tasks[task_id]["assignee"]["id"] assert payload["task"]["assignee"]["id"] == assignee_id - def test_webhook_update_task_label(self, tasks): - task_id, org_id = next( - ( - (task["id"], task["organization"]) - for task in tasks - if task["project_id"] is None and task["organization"] is not None - ) - ) - - webhook_id = create_webhook(["update:task"], "organization", org_id=org_id)["id"] - - patch_data = {"labels": [{"name": "new_label"}]} - response = patch_method("admin1", f"tasks/{task_id}", patch_data, org_id=org_id) - assert response.status_code == HTTPStatus.OK - - deliveries, payload = get_deliveries(webhook_id=webhook_id) - - assert deliveries["count"] == 1 - assert ( - payload["before_update"]["labels"]["count"] - == tasks[task_id]["labels"]["count"] - == payload["task"]["labels"]["count"] - 1 - ) - def test_webhook_create_and_delete_task(self, organizations): org_id = list(organizations)[0]["id"] events = ["create:task", "delete:task"] @@ -376,7 +330,7 @@ def test_webhook_create_and_delete_task(self, organizations): create_payload["task"], task, ignore_order=True, - exclude_paths=["root['updated_date']"], + exclude_paths=["root['updated_date']", "root['labels']"], ) == {} ) @@ -385,7 +339,7 @@ def test_webhook_create_and_delete_task(self, organizations): delete_payload["task"], task, ignore_order=True, - exclude_paths=["root['updated_date']"], + exclude_paths=["root['updated_date']", "root['labels']"], ) == {} ) @@ -539,7 +493,7 @@ def test_webhook_create_and_delete_issue(self, organizations, jobs, tasks): create_payload["issue"], issue, ignore_order=True, - exclude_paths=["root['updated_date']"], + exclude_paths=["root['updated_date']", "root['comments']"], ) == {} ) @@ -548,7 +502,7 @@ def test_webhook_create_and_delete_issue(self, organizations, jobs, tasks): delete_payload["issue"], issue, ignore_order=True, - exclude_paths=["root['updated_date']"], + exclude_paths=["root['updated_date']", "root['comments']"], ) == {} ) @@ -597,7 +551,7 @@ def test_webhook_delete_membership(self, memberships): payload["membership"], membership, ignore_order=True, - exclude_paths=["root['updated_date']"], + exclude_paths=["root['updated_date']", "root['invitation']"], ) == {} ) diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 7f699e739927..28989fbf4849 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -7492,85 +7492,6 @@ "organization": 1 } }, -{ - "model": "webhooks.webhookdelivery", - "pk": 1, - "fields": { - "webhook": 2, - "event": "update:job", - "status_code": 200, - "redelivery": false, - "created_date": "2022-11-03T13:57:26.380Z", - "updated_date": "2022-11-03T13:57:26.908Z", - "changed_fields": "state", - "request": { - "job": { - "id": 14, - "url": "http://localhost:8080/api/jobs/14", - "mode": "annotation", - "stage": "annotation", - "state": "in progress", - "labels": [ - { - "id": 6, - "name": "person", - "type": "any", - "color": "#c06060", - "sublabels": [], - "attributes": [], - "has_parent": false - }, - { - "id": 5, - "name": "car", - "type": "any", - "color": "#2080c0", - "sublabels": [], - "attributes": [ - { - "id": 1, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ], - "mutable": false, - "input_type": "select", - "default_value": "mazda" - } - ], - "has_parent": false - } - ], - "status": "annotation", - "task_id": 9, - "assignee": null, - "dimension": "2d", - "project_id": 1, - "stop_frame": 19, - "bug_tracker": "", - "start_frame": 15, - "updated_date": "2022-11-03T13:57:26.346824Z", - "data_chunk_size": 72, - "data_compressed_chunk_type": "imageset" - }, - "event": "update:job", - "sender": { - "id": 1, - "url": "http://localhost:8080/api/users/1", - "username": "admin1", - "last_name": "First", - "first_name": "Admin" - }, - "webhook_id": 2, - "before_update": { - "state": "new" - } - }, - "response": "\n\n\n Example Domain\n\n \n \n \n \n\n\n\n
\n

Example Domain

\n

This domain is for use in illustrative examples in documents. You may use this\n domain in literature without prior coordination or asking for permission.

\n

More information...

\n
\n\n\n" - } -}, { "model": "admin.logentry", "pk": 1, diff --git a/tests/python/shared/assets/webhooks.json b/tests/python/shared/assets/webhooks.json index 10676d528eb5..5b957c0fe66b 100644 --- a/tests/python/shared/assets/webhooks.json +++ b/tests/python/shared/assets/webhooks.json @@ -91,8 +91,6 @@ ], "id": 2, "is_active": true, - "last_delivery_date": "2022-11-03T13:57:26.908000Z", - "last_status": 200, "organization": null, "owner": { "first_name": "Business", diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index 5c2992817ce8..79071e736c3d 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -260,7 +260,8 @@ def get_server_image_tag(): def docker_compose(dc_files, cvat_root_dir): return [ - "docker-compose", + "docker", + "compose", f"--project-name={PREFIX}", # use compatibility mode to have fixed names for containers (with underscores) # https://github.com/docker/compose#about-update-and-backward-compatibility @@ -355,8 +356,9 @@ def local_start(start, stop, dumpdb, cleanup, rebuild, cvat_root_dir, cvat_db_di stop_services(dc_files, cvat_root_dir) pytest.exit("All testing containers are stopped", returncode=0) - if not any( - [cn in [f"{PREFIX}_cvat_server_1", f"{PREFIX}_cvat_db_1"] for cn in running_containers()] + if ( + not any(set(running_containers()) & {f"{PREFIX}_cvat_server_1", f"{PREFIX}_cvat_db_1"}) + or rebuild ): start_services(dc_files, rebuild, cvat_root_dir)