From 22625975af427796f027119248ac7addbbd15638 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 31 Jan 2019 14:33:09 +0300 Subject: [PATCH] Improved POST for api/v1/tasks --- cvat/apps/engine/admin.py | 3 +- .../migrations/0018_auto_20190131_1213.py | 25 +++++++++ .../migrations/0019_auto_20190131_1317.py | 28 ++++++++++ .../migrations/0020_remove_task_path.py | 17 +++++++ cvat/apps/engine/models.py | 48 ++++++++--------- cvat/apps/engine/serializers.py | 51 +++++++++++-------- cvat/apps/engine/task.py | 3 -- cvat/apps/engine/views.py | 6 +++ 8 files changed, 128 insertions(+), 53 deletions(-) create mode 100644 cvat/apps/engine/migrations/0018_auto_20190131_1213.py create mode 100644 cvat/apps/engine/migrations/0019_auto_20190131_1317.py create mode 100644 cvat/apps/engine/migrations/0020_remove_task_path.py diff --git a/cvat/apps/engine/admin.py b/cvat/apps/engine/admin.py index 9cc6599c71b..55786f89e45 100644 --- a/cvat/apps/engine/admin.py +++ b/cvat/apps/engine/admin.py @@ -56,8 +56,7 @@ def has_module_permission(self, request): class TaskAdmin(admin.ModelAdmin): date_hierarchy = 'updated_date' - readonly_fields = ('size', 'path', 'created_date', 'updated_date', - 'overlap', 'flipped') + readonly_fields = ('size', 'created_date', 'updated_date', 'overlap', 'flipped') list_display = ('name', 'mode', 'owner', 'assignee', 'created_date', 'updated_date') search_fields = ('name', 'mode', 'owner__username', 'owner__first_name', 'owner__last_name', 'owner__email', 'assignee__username', 'assignee__first_name', diff --git a/cvat/apps/engine/migrations/0018_auto_20190131_1213.py b/cvat/apps/engine/migrations/0018_auto_20190131_1213.py new file mode 100644 index 00000000000..ece220a6b0a --- /dev/null +++ b/cvat/apps/engine/migrations/0018_auto_20190131_1213.py @@ -0,0 +1,25 @@ +# Generated by Django 2.1.5 on 2019-01-31 09:13 + +import cvat.apps.engine.models +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0017_auto_20190130_2255'), + ] + + operations = [ + migrations.AlterField( + model_name='clientfile', + name='path', + field=models.FileField(storage=django.core.files.storage.FileSystemStorage, upload_to=cvat.apps.engine.models.upload_path_handler), + ), + migrations.AlterField( + model_name='task', + name='status', + field=models.CharField(choices=[('ANNOTATION', 'annotation'), ('VALIDATION', 'validation'), ('COMPLETED', 'completed')], default=cvat.apps.engine.models.StatusChoice('annotation'), max_length=32), + ), + ] diff --git a/cvat/apps/engine/migrations/0019_auto_20190131_1317.py b/cvat/apps/engine/migrations/0019_auto_20190131_1317.py new file mode 100644 index 00000000000..969ea6266b7 --- /dev/null +++ b/cvat/apps/engine/migrations/0019_auto_20190131_1317.py @@ -0,0 +1,28 @@ +# Generated by Django 2.1.5 on 2019-01-31 10:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0018_auto_20190131_1213'), + ] + + operations = [ + migrations.RenameField( + model_name='clientfile', + old_name='path', + new_name='file', + ), + migrations.RenameField( + model_name='remotefile', + old_name='path', + new_name='file', + ), + migrations.RenameField( + model_name='serverfile', + old_name='path', + new_name='file', + ), + ] diff --git a/cvat/apps/engine/migrations/0020_remove_task_path.py b/cvat/apps/engine/migrations/0020_remove_task_path.py new file mode 100644 index 00000000000..cd996d30cca --- /dev/null +++ b/cvat/apps/engine/migrations/0020_remove_task_path.py @@ -0,0 +1,17 @@ +# Generated by Django 2.1.5 on 2019-01-31 10:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0019_auto_20190131_1317'), + ] + + operations = [ + migrations.RemoveField( + model_name='task', + name='path', + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 46f819d9870..0ade948973b 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -15,13 +15,10 @@ import csv import re import os +import sys -fs = FileSystemStorage() -def upload_path_handler(instance, filename): - return os.path.join(instance.get_upload_dirname(), filename) - -class StatusChoice(Enum): +class StatusChoice(str, Enum): ANNOTATION = 'annotation' VALIDATION = 'validation' COMPLETED = 'completed' @@ -30,9 +27,6 @@ class StatusChoice(Enum): def choices(self): return tuple((x.name, x.value) for x in self) - def __str__(self): - return self.value - class SafeCharField(models.CharField): def get_prep_value(self, value): value = super().get_prep_value(value) @@ -43,7 +37,6 @@ def get_prep_value(self, value): class Task(models.Model): name = SafeCharField(max_length=256) size = models.PositiveIntegerField() - path = models.CharField(max_length=256) mode = models.CharField(max_length=32) owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="owners") @@ -53,64 +46,67 @@ class Task(models.Model): created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now_add=True) overlap = models.PositiveIntegerField(default=0) - segment_size = models.PositiveIntegerField() + segment_size = models.PositiveIntegerField(default=sys.maxsize) z_order = models.BooleanField(default=False) flipped = models.BooleanField(default=False) # FIXME: remote source field source = SafeCharField(max_length=256, default="unknown") - status = models.CharField(max_length=32, default=StatusChoice.ANNOTATION) + status = models.CharField(max_length=32, choices=StatusChoice.choices(), + default=StatusChoice.ANNOTATION) # Extend default permission model class Meta: default_permissions = () def get_upload_dirname(self): - return os.path.join(self.path, ".upload") + return os.path.join(self.get_task_dirname(), ".upload") def get_data_dirname(self): - return os.path.join(self.path, "data") + return os.path.join(self.get_task_dirname(), "data") def get_dump_path(self): name = re.sub(r'[\\/*?:"<>|]', '_', self.name) - return os.path.join(self.path, "{}.xml".format(name)) + return os.path.join(self.get_task_dirname(), "{}.xml".format(name)) def get_log_path(self): - return os.path.join(self.path, "task.log") + return os.path.join(self.get_task_dirname(), "task.log") def get_client_log_path(self): - return os.path.join(self.path, "client.log") + return os.path.join(self.get_task_dirname(), "client.log") def get_image_meta_cache_path(self): - return os.path.join(self.path, "image_meta.cache") - - def set_task_dirname(self, path): - self.path = path - self.save(update_fields=['path']) + return os.path.join(self.get_task_dirname(), "image_meta.cache") def get_task_dirname(self): - return self.path + return os.path.join(settings.DATA_ROOT, str(self.id)) def __str__(self): return self.name +def upload_path_handler(instance, filename): + return os.path.join(instance.get_upload_dirname(), filename) + +# For client files which the user is uploaded class ClientFile(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE) - path = models.FileField(upload_to=upload_path_handler, - storage=fs) + file = models.FileField(upload_to=upload_path_handler, + storage=FileSystemStorage) class Meta: default_permissions = () +# For server files on the mounted share class ServerFile(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE) - path = models.CharField(max_length=1024) + file = models.CharField(max_length=1024) class Meta: default_permissions = () +# For URLs class RemoteFile(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE) - path = models.CharField(max_length=1024) + file = models.CharField(max_length=1024) class Meta: default_permissions = () diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 544c44e3932..35f4315e24e 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -7,6 +7,8 @@ Segment, ClientFile, ServerFile, RemoteFile) from django.contrib.auth.models import User, Group +import os +import shutil class AttributeSerializer(serializers.ModelSerializer): class Meta: @@ -19,14 +21,6 @@ class Meta: model = Label fields = ('id', 'name', 'attributes') - def create(self, validated_data): - attributes = validated_data.pop('attributes') - label = Label.objects.create(**validated_data) - for attr in attributes: - AttributeSpec.objects.create(label=label, **attr) - - return label - class JobSerializer(serializers.ModelSerializer): task_id = serializers.ReadOnlyField(source="segment.task.id") start_frame = serializers.ReadOnlyField(source="segment.start_frame") @@ -57,25 +51,25 @@ class Meta: fields = ('path', ) def to_internal_value(self, data): - return { "path": data } + return { 'file': data } class ServerFileSerializer(serializers.ModelSerializer): class Meta: model = ServerFile - fields = ('path', ) + fields = ('file', ) def to_internal_value(self, data): - return { "path": data } + return { 'file': data } class RemoteFileSerializer(serializers.ModelSerializer): class Meta: model = RemoteFile - fields = ('path', ) + fields = ('file', ) def to_internal_value(self, data): - return { "path": data } + return { 'file' : data } class TaskSerializer(serializers.ModelSerializer): labels = LabelSerializer(many=True, source='label_set') @@ -95,26 +89,39 @@ class Meta: 'server_files', 'client_files', 'remote_files') read_only_fields = ('size', 'mode', 'created_date', 'updated_date', 'overlap', 'status', 'segment_size') + ordering = ['-id'] def create(self, validated_data): labels = validated_data.pop('label_set') client_files = validated_data.pop('clientfile_set') server_files = validated_data.pop('serverfile_set') remote_files = validated_data.pop('remotefile_set') - task = Task.objects.create(**validated_data) + db_task = Task.objects.create(size=0, **validated_data) for label in labels: - Label.objects.create(task=task, **label) + attributes = label.pop('attributespec_set') + db_label = Label.objects.create(task=db_task, **label) + for attr in attributes: + AttributeSpec.objects.create(label=db_label, **attr) + + for file in client_files: + ClientFile.objects.create(task=db_task, file=file) + + for file in server_files: + ServerFile.objects.create(task=db_task, file=file) - for path in client_files: - ClientFile.objects.create(task=task, path=path) + for file in remote_files: + RemoteFile.objects.create(task=db_task, file=file) - for path in server_files: - ServerFile.objects.create(task=task, path=path) + task_path = db_task.get_task_dirname() + if os.path.isdir(task_path): + shutil.rmtree(task_path) - for path in remote_files: - RemoteFile.objects.create(task=task, path=path) + upload_dir = db_task.get_upload_dirname() + os.makedirs(upload_dir) + output_dir = db_task.get_data_dirname() + os.makedirs(output_dir) - return task + return db_task class UserSerializer(serializers.ModelSerializer): groups = serializers.SlugRelatedField(many=True, diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index db7c7e35624..9cc1cfcab77 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -42,12 +42,9 @@ def create_empty(params): db_task.name = params['task_name'] db_task.bug_tracker = params['bug_tracker_link'] - db_task.path = "" db_task.size = 0 db_task.owner = params['owner'] db_task.save() - task_path = os.path.join(settings.DATA_ROOT, str(db_task.id)) - db_task.set_task_dirname(task_path) task_path = db_task.get_task_dirname() if os.path.isdir(task_path): diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 9132026866e..f2704641ed3 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -51,6 +51,12 @@ class TaskList(generics.ListCreateAPIView): queryset = Task.objects.all() serializer_class = TaskSerializer + def perform_create(self, serializer): + if self.request.data['owner']: + serializer.save() + else: + serializer.save(owner=self.request.user) + class TaskDetail(generics.RetrieveUpdateDestroyAPIView): queryset = Task.objects.all() serializer_class = TaskSerializer