From 457358b6ba9509d19dd0ad4db5feb0006b8680e4 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Sat, 28 Aug 2021 05:30:28 +0300 Subject: [PATCH 01/57] temp --- cvat-ui/src/actions/import-actions.ts | 55 +++++++++++ .../import-dataset-modal.tsx | 27 ++++++ cvat-ui/src/reducers/import-reducer.ts | 21 ++++ cvat-ui/src/reducers/interfaces.ts | 9 ++ cvat-ui/src/reducers/root-reducer.ts | 1 + cvat/apps/dataset_manager/bindings.py | 11 ++- cvat/apps/dataset_manager/formats/coco.py | 8 +- cvat/apps/dataset_manager/formats/registry.py | 4 +- cvat/apps/dataset_manager/project.py | 37 ++++++- cvat/apps/engine/serializers.py | 3 + cvat/apps/engine/views.py | 96 +++++++++++++++++-- 11 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 cvat-ui/src/actions/import-actions.ts create mode 100644 cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx create mode 100644 cvat-ui/src/reducers/import-reducer.ts diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts new file mode 100644 index 000000000000..47a0a6062e5b --- /dev/null +++ b/cvat-ui/src/actions/import-actions.ts @@ -0,0 +1,55 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ActionCreator, AnyAction, Dispatch } from 'redux'; +import { ThunkAction } from 'redux-thunk'; +import { getCVATStore } from 'cvat-store'; +import { createAction } from 'utils/redux'; +import { CombinedState } from 'reducers/interfaces'; + +export enum ImportActionType { + OPEN_IMPORT_MODAL = 'OPEN_IMPORT_MODAL', + CLOSE_IMPORT_MODAL = 'OPEN_IMPORT_MODAL', + IMPORT_DATASET = 'IMPORT_DATASET', + IMPORT_DATASET_SUCCESS = 'IMPORT_DATASET_SUCCESS', + IMPORT_DATASET_FAILED = 'IMPORT_DATASET_FAILED', +} + +export const importActions = { + openImportModal: (instance: any) => createAction(ImportActionType.OPEN_IMPORT_MODAL, { instance }), + closeImportModal: () => createAction(ImportActionType.CLOSE_IMPORT_MODAL), + importDataset: (instance: any, format: string) => + createAction(ImportActionType.IMPORT_DATASET, { instance, format }), + importDatasetSuccess: (instance: any, format: string) => + createAction(ImportActionType.IMPORT_DATASET_SUCCESS, { instance, format }), + importDatasetFailed: (instance: any, format: string, error: any) => + createAction(ImportActionType.IMPORT_DATASET_FAILED, { + instance, + format, + error, + }), +}; + +export function importDatasetAsync( + instance: any, + format: string, + file: File, +): ThunkAction, {}, {}, AnyAction> { + return async (dispatch: ActionCreator): Promise => { + try { + const store = getCVATStore(); + const state: CombinedState = store.getState(); + if (state.import.projects[instance.id]) { + throw Error('Only one importing of dataset allowed at the same time') + } + dispatch(importActions.importDataset(instance, format)); + await instance.dataset.import(file, format); + } catch (error) { + dispatch(importActions.importDatasetFailed(instance, format, error)); + return; + } + + dispatch(importActions.importDatasetSuccess(instance, format)); + }; +} diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx new file mode 100644 index 000000000000..213345656ee3 --- /dev/null +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -0,0 +1,27 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import React from 'react'; +import { useSelector } from 'react-redux'; +import Modal from 'antd/lib/modal'; + +import { CombinedState } from 'reducers/interfaces'; + +function ImportDatasetModal(): JSX.Element { + const modalVisible = useSelector((state: CombinedState) => state.import.modalVisible); + + return ( + {}} + onOk={() => {}} + className={`cvat-modal-import-${'project'}`} + > +
+ + ); +} + +export default ImportDatasetModal; diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts new file mode 100644 index 000000000000..7a7c5b468be1 --- /dev/null +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -0,0 +1,21 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ImportActions, ImportActionTypes } from 'actions/import-actions'; +import getCore from 'cvat-core-wrapper'; +import deepCopy from 'utils/deep-copy'; + +import { ImportState } from './interfaces'; + +const core = getCore(); + +const defaultState: ImportState = { + projects: {}, + instance: null, + modalVisible: false, +}; + +export default (state: ImportState = defaultState, action: ImportActions): ImportState => { + switch +} \ No newline at end of file diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index ef11a8207463..6ffd0cd15f24 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -115,6 +115,14 @@ export interface ExportState { modalVisible: boolean; } +export interface ImportState { + projects: { + [pid: number]: string[]; + }; + instance: any; + modalVisible: boolean; +} + export interface FormatsState { annotationFormats: any; fetching: boolean; @@ -633,6 +641,7 @@ export interface CombinedState { shortcuts: ShortcutsState; review: ReviewState; export: ExportState; + import: ImportState; } export enum DimensionType { diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index b1219b7a0b23..cb09f6e783b4 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -36,5 +36,6 @@ export default function createRootReducer(): Reducer { userAgreements: userAgreementsReducer, review: reviewReducer, export: exportReducer, + import: import }); } diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 2062e0333fba..12d2bcb33eb2 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -11,6 +11,7 @@ from django.utils import timezone +from datumaro.components.dataset import Dataset import datumaro.components.extractor as datumaro from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.models import AttributeType, ShapeType, Project, Task, Label, DimensionType, Image as Img @@ -515,7 +516,7 @@ class ProjectData(InstanceLabelData): Tag.__new__.__defaults__ = (0, ) Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) - def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): + def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str = '', create_callback: Callable = None): self._annotation_irs = annotation_irs self._db_project = db_project self._db_tasks: OrderedDict[int, Task] = OrderedDict( @@ -1261,3 +1262,11 @@ def import_dm_annotations(dm_dataset, task_data): except Exception as e: raise CvatImportError("Image {}: can't import annotation " "#{} ({}): {}".format(item.id, idx, ann.type.name, e)) + +def import_labels_to_project(project_annotation, dataset): + pass + +def load_dataset_data(project_annotation, dataset): + import_labels_to_project(project_annotation, dataset) + + diff --git a/cvat/apps/dataset_manager/formats/coco.py b/cvat/apps/dataset_manager/formats/coco.py index 927df2de567a..925c9d32600f 100644 --- a/cvat/apps/dataset_manager/formats/coco.py +++ b/cvat/apps/dataset_manager/formats/coco.py @@ -13,7 +13,6 @@ from .registry import dm_env, exporter, importer - @exporter(name='COCO', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): dataset = Dataset.from_extractors(GetCVATDataExtractor( @@ -25,14 +24,17 @@ def _export(dst_file, instance_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='COCO', ext='JSON, ZIP', version='1.0') -def _import(src_file, instance_data): +def _import(src_file, instance_data, load_data_callback=None): if zipfile.is_zipfile(src_file): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) + dataset = Dataset.import_from(tmp_dir, 'coco', env=dm_env) + if load_data_callback is not None: + load_data_callback(dataset) import_dm_annotations(dataset, instance_data) else: dataset = Dataset.import_from(src_file.name, 'coco_instances', env=dm_env) - import_dm_annotations(dataset, instance_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/registry.py b/cvat/apps/dataset_manager/formats/registry.py index 959127ca0176..868afb8cd97f 100644 --- a/cvat/apps/dataset_manager/formats/registry.py +++ b/cvat/apps/dataset_manager/formats/registry.py @@ -17,11 +17,11 @@ class _Format: ENABLED = True class Exporter(_Format): - def __call__(self, dst_file, task_data, **options): + def __call__(self, dst_file, instance_data, **options): raise NotImplementedError() class Importer(_Format): - def __call__(self, src_file, task_data, **options): + def __call__(self, src_file, instance_data, load_data_callback=None, **options): raise NotImplementedError() def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled, dimension=DimensionType.DIM_2D): diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py index 866a75d47e8b..52925bf80854 100644 --- a/cvat/apps/dataset_manager/project.py +++ b/cvat/apps/dataset_manager/project.py @@ -10,8 +10,8 @@ from cvat.apps.dataset_manager.task import TaskAnnotation from .annotation import AnnotationIR -from .bindings import ProjectData -from .formats.registry import make_exporter +from .bindings import ProjectData, load_dataset_data +from .formats.registry import make_exporter, make_importer def export_project(project_id, dst_file, format_name, server_url=None, save_images=False): @@ -21,18 +21,19 @@ def export_project(project_id, dst_file, format_name, # more dump request received at the same time: # https://github.com/opencv/cvat/issues/217 with transaction.atomic(): - project = ProjectAnnotation(project_id) + project = ProjectAnnotationAndData(project_id) project.init_from_db() exporter = make_exporter(format_name) with open(dst_file, 'wb') as f: project.export(f, exporter, host=server_url, save_images=save_images) -class ProjectAnnotation: +class ProjectAnnotationAndData: def __init__(self, pk: int): self.db_project = models.Project.objects.get(id=pk) self.db_tasks = models.Task.objects.filter(project__id=pk).order_by('id') + self.task_annotations: dict[int, TaskAnnotation] = dict() self.annotation_irs: dict[int, AnnotationIR] = dict() def reset(self): @@ -57,6 +58,7 @@ def init_from_db(self): for task in self.db_tasks: annotation = TaskAnnotation(pk=task.id) annotation.init_from_db() + self.task_annotations[task.id] = annotation self.annotation_irs[task.id] = annotation.ir_data def export(self, dst_file: str, exporter: Callable, host: str='', **options): @@ -66,6 +68,31 @@ def export(self, dst_file: str, exporter: Callable, host: str='', **options): host=host ) exporter(dst_file, project_data, **options) + + def load_dataset_data(self, *args, **kwargs): + load_dataset_data(self, *args, **kwargs) + + + def import_dataset(self, dataset_file, importer): + project_data = ProjectData( + annotation_irs=self.annotation_irs, + db_project=self.db_project, + ) + + + + importer(dataset_file, project_data, self.load_dataset_data) + self.create(None) + @property def data(self) -> dict: - raise NotImplementedError() \ No newline at end of file + raise NotImplementedError() + +@transaction.atomic +def import_dataset_as_project(project_id, dataset_file, format_name): + project = ProjectAnnotationAndData(project_id) + project.init_from_db() + + importer = make_importer(format_name) + with open(dataset_file, 'rb') as f: + project.import_dataset(f, importer) \ No newline at end of file diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index f50e799bf427..403f3d1fa16f 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -716,6 +716,9 @@ class LogEventSerializer(serializers.Serializer): class AnnotationFileSerializer(serializers.Serializer): annotation_file = serializers.FileField() +class DatasetFileSerializer(serializers.Serializer): + dataset_file = serializers.FileField() + class TaskFileSerializer(serializers.Serializer): task_file = serializers.FileField() diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index afe9ef504fa8..d42c13db57c4 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -57,7 +57,7 @@ LogEventSerializer, ProjectSerializer, ProjectSearchSerializer, ProjectWithoutTaskSerializer, RqStatusSerializer, TaskSerializer, UserSerializer, PluginsSerializer, ReviewSerializer, CombinedReviewSerializer, IssueSerializer, CombinedIssueSerializer, CommentSerializer, - CloudStorageSerializer, BaseCloudStorageSerializer, TaskFileSerializer,) + CloudStorageSerializer, BaseCloudStorageSerializer, TaskFileSerializer, DatasetFileSerializer) from utils.dataset_manifest import ImageManifestManager from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.backup import import_task @@ -310,7 +310,7 @@ def tasks(self, request, pk): type=openapi.TYPE_STRING, required=False), openapi.Parameter('action', in_=openapi.IN_QUERY, description='Used to start downloading process after annotation file had been created', - type=openapi.TYPE_STRING, required=False, enum=['download']) + type=openapi.TYPE_STRING, required=False, enum=['download', 'import_status']) ], responses={'202': openapi.Response(description='Exporting has been started'), '201': openapi.Response(description='Output file is ready for downloading'), @@ -322,15 +322,63 @@ def tasks(self, request, pk): url_path='dataset') def dataset_export(self, request, pk): db_project = self.get_object() # force to call check_object_permissions + action = request.query_params.get("action", "").lower() + + if action in ("import_status",): + queue = django_rq.get_queue("default") + rq_job = queue.fetch_job(f"/api/v1/project/{pk}/dataset_import") + if rq_job.is_finished: + os.close(rq_job.meta['tmp_file_descriptor']) + os.remove(rq_job.meta['tmp_file']) + rq_job.delete() + return Response(status=status.HTTP_201_CREATED) + elif rq_job.is_failed: + os.close(rq_job.meta['tmp_file_descriptor']) + os.remove(rq_job.meta['tmp_file']) + rq_job.delete() + #TODO: Should we check CVATImportError here? + return Response( + data=str(rq_job.exc_info), + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + else: + format_name = request.query_params.get("format", "") + return _export_annotations( + db_instance=db_project, + rq_id="/api/v1/project/{}/dataset/{}".format(pk, format_name), + request=request, + action=action, + callback=dm.views.export_project_as_dataset, + format_name=format_name, + filename=request.query_params.get("filename", "").lower(), + ) + + @swagger_auto_schema(method='post', operation_summary='Import dataset in specific format as a project', + manual_parameters=[ + openapi.Parameter('format', openapi.IN_QUERY, + description="Desired dataset format name\nYou can get the list of supported formats at:\n/server/annotation/formats", + type=openapi.TYPE_STRING, required=True) + ], + responses={'202': openapi.Response(description='Exporting has been started'), + '400': openapi.Response(description='Failed to import dataset'), + '405': openapi.Response(description='Format is not available'), + } + ) + @action(detail=True, methods=['POST'], serializer_class=None, url_path='dataset') + def dataset_import(self, request, pk): + project: Project = self.get_object() # force to call check_object_permissions format_name = request.query_params.get("format", "") - return _export_annotations(db_instance=db_project, - rq_id="/api/v1/project/{}/dataset/{}".format(pk, format_name), + + if project.tasks.count(): + raise ValidationError("Cannot import dataset in non empty project") + + return _import_project_dataset( request=request, - action=request.query_params.get("action", "").lower(), - callback=dm.views.export_project_as_dataset, + rq_id=f"/api/v1/project/{pk}/dataset_import", + rq_func=dm.project.import_dataset_as_project, + pk=pk, format_name=format_name, - filename=request.query_params.get("filename", "").lower(), ) @swagger_auto_schema(method='get', operation_summary='Method allows to download project annotations', @@ -1459,8 +1507,8 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) queue = django_rq.get_queue("default") - rq_job = queue.fetch_job(rq_id) + if rq_job: last_instance_update_time = timezone.localtime(db_instance.updated_date) if isinstance(db_instance, Project): @@ -1511,4 +1559,36 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba result_ttl=ttl, failure_ttl=ttl) return Response(status=status.HTTP_202_ACCEPTED) +def _import_project_dataset(request, rq_id, rq_func, pk, format_name): + format_desc = {f.DISPLAY_NAME: f + for f in dm.views.get_import_formats()}.get(format_name) + if format_desc is None: + raise serializers.ValidationError( + "Unknown input format '{}'".format(format_name)) + + queue = django_rq.get_queue("default") + rq_job = queue.fetch_job(rq_id) + + if not rq_job: + serializer = DatasetFileSerializer(data=request.data) + if serializer.is_valid(raise_exception=True): + dataset_file = serializers.validated_data['dataset_file'] + fd, filename = mkstemp(prefix='cvat_{}'.format(pk)) + with open(filename, 'wb+') as f: + for chunk in dataset_file.chunks(): + f.write(chunk) + + #TODO: should we check zip here? + rq_job = queue.enqueue_call( + func=rq_func, + args=(pk, filename, format_name), + job_id=rq_id + ) + rq_job.meta['tmp_file'] = filename + rq_job.meta['tmp_file_descriptor'] = fd + rq_job.save_meta() + else: + raise ValidationError("Import job already exists") + + From ec977dd4cc54f49eb2dc3560193fbf761a6d5a93 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 3 Sep 2021 12:36:55 +0300 Subject: [PATCH 02/57] drafgt ui implementation --- cvat-core/src/annotations.js | 12 ++ cvat-core/src/project-implementation.js | 10 +- cvat-core/src/project.js | 10 ++ cvat-core/src/server-proxy.js | 45 ++++++- cvat-ui/src/actions/import-actions.ts | 57 ++++----- .../import-dataset-modal.tsx | 110 +++++++++++++++++- .../components/projects-page/actions-menu.tsx | 9 +- .../projects-page/projects-page.tsx | 2 + cvat-ui/src/reducers/import-reducer.ts | 46 +++++++- cvat-ui/src/reducers/interfaces.ts | 4 +- cvat-ui/src/reducers/root-reducer.ts | 3 +- cvat/apps/engine/views.py | 4 +- 12 files changed, 256 insertions(+), 56 deletions(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 07ce90f9e2f0..53dc70b4ba99 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -284,6 +284,17 @@ return result; } + async function importDataset(instance, format, file) { + if (!(format instanceof String || typeof format === 'string')) { + throw new ArgumentError('Format must be a string'); + } + if (!(instance instanceof Project)) { + throw new ArgumentError('Instance should ne a Project isntance'); + } + // TODO: check file + return serverProxy.projects.importDataset(instance.id, format, file); + } + function undoActions(session, count) { const sessionType = session instanceof Task ? 'task' : 'job'; const cache = getCache(sessionType); @@ -366,6 +377,7 @@ importAnnotations, exportAnnotations, exportDataset, + importDataset, undoActions, redoActions, freezeHistory, diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js index c5bb2387099d..5a736f7fd8ce 100644 --- a/cvat-core/src/project-implementation.js +++ b/cvat-core/src/project-implementation.js @@ -7,7 +7,7 @@ const { getPreview } = require('./frames'); const { Project } = require('./project'); - const { exportDataset } = require('./annotations'); + const { exportDataset, importDataset } = require('./annotations'); function implementProject(projectClass) { projectClass.prototype.save.implementation = async function () { @@ -61,11 +61,17 @@ }; projectClass.prototype.annotations.exportDataset.implementation = async function ( - format, saveImages, customName, + format, + saveImages, + customName, ) { const result = exportDataset(this, format, customName, saveImages); return result; }; + projectClass.prototype.annotations.importDataset.implementation = async function (format, file) { + const result = importDataset(this, format, file); + return result; + }; return projectClass; } diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index 7e324498b95f..5f9e625cce6f 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -270,6 +270,7 @@ // So, we need return it this.annotations = { exportDataset: Object.getPrototypeOf(this).annotations.exportDataset.bind(this), + importDataset: Object.getPrototypeOf(this).annotations.importDataset.bind(this), }; } @@ -336,6 +337,15 @@ ); return result; }, + async importDataset(format, file) { + const result = await PluginRegistry.apiWrapper.call( + this, + Project.prototype.annotations.importDataset, + format, + file, + ); + return result; + }, }, writable: true, }), diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 3d914ce9db7a..cc14fd13abc4 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -500,6 +500,46 @@ }; } + async function importDataset(id, format, file) { + const { backendAPI } = config; + const url = `${backendAPI}/project/${id}/dataset`; + + const formData = new FormData(); + formData.append('dataset_file', file); + + return new Promise((resolve, reject) => { + async function request() { + try { + const response = await Axios.get(`${url}?action=import_status`, { + proxy: config.proxy, + }); + if (response.status === 202) { + setTimeout(request, 3000); + } else if (response.status === 201) { + resolve(); + } else { + reject(generateError(response)); + } + } catch (error) { + if (error.response.status === 404) { + try { + await Axios.post(`${url}?format=${format}`, formData, { + proxy: config.proxy, + }); + setTimeout(request, 3000); + } catch (_error) { + reject(generateError(error)); + } + } else { + reject(generateError(error)); + } + } + } + + setTimeout(request); + }); + } + async function exportTask(id) { const { backendAPI } = config; const url = `${backendAPI}/tasks/${id}`; @@ -1145,9 +1185,7 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - const predicate = () => ( - !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId - ); + const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { @@ -1212,6 +1250,7 @@ create: createProject, delete: deleteProject, exportDataset: exportDataset('projects'), + importDataset, }), writable: false, }, diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 47a0a6062e5b..c651589c20d4 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -2,54 +2,47 @@ // // SPDX-License-Identifier: MIT -import { ActionCreator, AnyAction, Dispatch } from 'redux'; -import { ThunkAction } from 'redux-thunk'; import { getCVATStore } from 'cvat-store'; -import { createAction } from 'utils/redux'; +import { createAction, ActionUnion, ThunkAction } from 'utils/redux'; import { CombinedState } from 'reducers/interfaces'; -export enum ImportActionType { +export enum ImportActionTypes { OPEN_IMPORT_MODAL = 'OPEN_IMPORT_MODAL', - CLOSE_IMPORT_MODAL = 'OPEN_IMPORT_MODAL', + CLOSE_IMPORT_MODAL = 'CLOSE_IMPORT_MODAL', IMPORT_DATASET = 'IMPORT_DATASET', IMPORT_DATASET_SUCCESS = 'IMPORT_DATASET_SUCCESS', IMPORT_DATASET_FAILED = 'IMPORT_DATASET_FAILED', } export const importActions = { - openImportModal: (instance: any) => createAction(ImportActionType.OPEN_IMPORT_MODAL, { instance }), - closeImportModal: () => createAction(ImportActionType.CLOSE_IMPORT_MODAL), + openImportModal: (instance: any) => createAction(ImportActionTypes.OPEN_IMPORT_MODAL, { instance }), + closeImportModal: () => createAction(ImportActionTypes.CLOSE_IMPORT_MODAL), importDataset: (instance: any, format: string) => - createAction(ImportActionType.IMPORT_DATASET, { instance, format }), + createAction(ImportActionTypes.IMPORT_DATASET, { instance, format }), importDatasetSuccess: (instance: any, format: string) => - createAction(ImportActionType.IMPORT_DATASET_SUCCESS, { instance, format }), - importDatasetFailed: (instance: any, format: string, error: any) => - createAction(ImportActionType.IMPORT_DATASET_FAILED, { + createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS, { instance, format }), + importDatasetFailed: (instance: any, error: any) => + createAction(ImportActionTypes.IMPORT_DATASET_FAILED, { instance, - format, error, }), }; -export function importDatasetAsync( - instance: any, - format: string, - file: File, -): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - try { - const store = getCVATStore(); - const state: CombinedState = store.getState(); - if (state.import.projects[instance.id]) { - throw Error('Only one importing of dataset allowed at the same time') - } - dispatch(importActions.importDataset(instance, format)); - await instance.dataset.import(file, format); - } catch (error) { - dispatch(importActions.importDatasetFailed(instance, format, error)); - return; +export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch) => { + try { + const store = getCVATStore(); + const state: CombinedState = store.getState(); + if (state.import.projects[instance.id]) { + throw Error('Only one importing of dataset allowed at the same time'); } + dispatch(importActions.importDataset(instance, format)); + await instance.annotations.importDataset(format, file); + } catch (error) { + dispatch(importActions.importDatasetFailed(instance, error)); + return; + } - dispatch(importActions.importDatasetSuccess(instance, format)); - }; -} + dispatch(importActions.importDatasetSuccess(instance, format)); +}; + +export type ImportActions = ActionUnion; diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index 213345656ee3..a433a7c1cff4 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -2,24 +2,122 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; -import { useSelector } from 'react-redux'; +import React, { useCallback, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; import Modal from 'antd/lib/modal'; +import Form from 'antd/lib/form'; +import Text from 'antd/lib/typography/Text'; +import Select from 'antd/lib/select'; +import Notification from 'antd/lib/notification'; +import message from 'antd/lib/message'; +import Upload, { RcFile } from 'antd/lib/upload'; +import { DownloadOutlined, InboxOutlined, LoadingOutlined } from '@ant-design/icons'; import { CombinedState } from 'reducers/interfaces'; +import { importActions, importDatasetAsync } from 'actions/import-actions'; + +type FormValues = { + selectedFormat: string | undefined; +}; function ImportDatasetModal(): JSX.Element { + const [form] = Form.useForm(); + const [file, setFile] = useState(null); const modalVisible = useSelector((state: CombinedState) => state.import.modalVisible); + const instance = useSelector((state: CombinedState) => state.import.instance); + const projects = useSelector((state: CombinedState) => state.import.projects); + const importers = useSelector((state: CombinedState) => state.formats.annotationFormats.loaders); + const dispatch = useDispatch(); + + const closeModal = (): void => { + form.resetFields(); + setFile(null); + dispatch(importActions.closeImportModal()); + }; + + const handleImport = useCallback( + (values: FormValues): void => { + if (file === null) { + Notification.error({ + message: 'No dataset file selected', + }); + return; + } + dispatch(importDatasetAsync(instance, values.selectedFormat as string, file)); + closeModal(); + Notification.info({ + message: 'Dataset export started', + description: `Dataset import was started for project #${instance?.id}. `, + className: 'cvat-notification-notice-import-project-start', + }); + }, + [instance?.id, file], + ); return ( {}} - onOk={() => {}} - className={`cvat-modal-import-${'project'}`} + onCancel={closeModal} + onOk={() => form.submit()} + className='cvat-modal-import-project' > -
+
+ + + + { + if (!['application/zip', 'application/x-zip-compressed'].includes(_file.type)) { + message.error('Only ZIP archive is supported'); + } else { + setFile(_file); + } + return false; + }} + onRemove={() => { + setFile(null); + }} + > +

+ +

+

Click or drag file to this area

+ +
); } diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index c85684ad6dbf..b68e18ca5790 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -7,8 +7,10 @@ import { useDispatch } from 'react-redux'; import Modal from 'antd/lib/modal'; import Menu from 'antd/lib/menu'; +import { MenuItem } from 'rc-menu'; import { deleteProjectAsync } from 'actions/projects-actions'; import { exportActions } from 'actions/export-actions'; +import { importActions } from 'actions/import-actions'; interface Props { projectInstance: any; @@ -37,11 +39,10 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( - dispatch(exportActions.openExportModal(projectInstance))} - > - Export project dataset + dispatch(exportActions.openExportModal(projectInstance))}> + Export dataset + dispatch(importActions.openImportModal(projectInstance))}>Import dataset
Delete diff --git a/cvat-ui/src/components/projects-page/projects-page.tsx b/cvat-ui/src/components/projects-page/projects-page.tsx index a2842348af50..cfc594bb98bc 100644 --- a/cvat-ui/src/components/projects-page/projects-page.tsx +++ b/cvat-ui/src/components/projects-page/projects-page.tsx @@ -12,6 +12,7 @@ import { CombinedState, ProjectsQuery } from 'reducers/interfaces'; import { getProjectsAsync } from 'actions/projects-actions'; import FeedbackComponent from 'components/feedback/feedback'; import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; +import ImportDatasetModal from 'components/import-dataset-modal/import-dataset-modal'; import EmptyListComponent from './empty-list'; import TopBarComponent from './top-bar'; import ProjectListComponent from './project-list'; @@ -57,6 +58,7 @@ export default function ProjectsPageComponent(): JSX.Element { {projectsCount ? : } +
); } diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index 7a7c5b468be1..bab7723837ec 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -3,13 +3,10 @@ // SPDX-License-Identifier: MIT import { ImportActions, ImportActionTypes } from 'actions/import-actions'; -import getCore from 'cvat-core-wrapper'; import deepCopy from 'utils/deep-copy'; import { ImportState } from './interfaces'; -const core = getCore(); - const defaultState: ImportState = { projects: {}, instance: null, @@ -17,5 +14,44 @@ const defaultState: ImportState = { }; export default (state: ImportState = defaultState, action: ImportActions): ImportState => { - switch -} \ No newline at end of file + switch (action.type) { + case ImportActionTypes.OPEN_IMPORT_MODAL: + return { + ...state, + modalVisible: true, + instance: action.payload.instance, + }; + case ImportActionTypes.CLOSE_IMPORT_MODAL: { + return { + ...state, + modalVisible: false, + instance: null, + }; + } + case ImportActionTypes.IMPORT_DATASET: { + const { instance, format } = action.payload; + const activities = deepCopy(state.projects); + + activities[instance.id] = format; + + return { + ...state, + projects: activities, + }; + } + case ImportActionTypes.IMPORT_DATASET_FAILED: + case ImportActionTypes.IMPORT_DATASET_SUCCESS: { + const { instance } = action.payload; + const activities = deepCopy(state.projects); + + delete activities[instance.id]; + + return { + ...state, + projects: activities, + }; + } + default: + return state; + } +}; diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 6ffd0cd15f24..44ea8d0c1f5b 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -117,7 +117,7 @@ export interface ExportState { export interface ImportState { projects: { - [pid: number]: string[]; + [pid: number]: string; }; instance: any; modalVisible: boolean; @@ -193,7 +193,7 @@ export interface Model { framework: string; description: string; type: string; - onChangeToolsBlockerState: (event:string) => void; + onChangeToolsBlockerState: (event: string) => void; tip: { message: string; gif: string; diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index cb09f6e783b4..4c98c9cd6a82 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -18,6 +18,7 @@ import shortcutsReducer from './shortcuts-reducer'; import userAgreementsReducer from './useragreements-reducer'; import reviewReducer from './review-reducer'; import exportReducer from './export-reducer'; +import importReducer from './import-reducer'; export default function createRootReducer(): Reducer { return combineReducers({ @@ -36,6 +37,6 @@ export default function createRootReducer(): Reducer { userAgreements: userAgreementsReducer, review: reviewReducer, export: exportReducer, - import: import + import: importReducer, }); } diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index d42c13db57c4..f57b8cc5e26c 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -328,7 +328,9 @@ def dataset_export(self, request, pk): if action in ("import_status",): queue = django_rq.get_queue("default") rq_job = queue.fetch_job(f"/api/v1/project/{pk}/dataset_import") - if rq_job.is_finished: + if rq_job is None: + return Response(status=status.HTTP_404_NOT_FOUND) + elif rq_job.is_finished: os.close(rq_job.meta['tmp_file_descriptor']) os.remove(rq_job.meta['tmp_file']) rq_job.delete() From 874bc2806cb1b3c51a47850d1b6e5e99c9cb26fe Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 7 Sep 2021 12:42:52 +0300 Subject: [PATCH 03/57] Added tooltip --- .../import-dataset-modal.tsx | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index a433a7c1cff4..3602fa15d196 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -12,7 +12,14 @@ import Notification from 'antd/lib/notification'; import message from 'antd/lib/message'; import Upload, { RcFile } from 'antd/lib/upload'; -import { DownloadOutlined, InboxOutlined, LoadingOutlined } from '@ant-design/icons'; +import { + DownloadOutlined, + InboxOutlined, + LoadingOutlined, + QuestionCircleFilled, +} from '@ant-design/icons'; + +import CVATTooltip from 'components/common/cvat-tooltip'; import { CombinedState } from 'reducers/interfaces'; import { importActions, importDatasetAsync } from 'actions/import-actions'; @@ -56,7 +63,19 @@ function ImportDatasetModal(): JSX.Element { return ( + Import dataset to project + + + + + )} visible={modalVisible} onCancel={closeModal} onOk={() => form.submit()} @@ -76,7 +95,10 @@ function ImportDatasetModal(): JSX.Element { {importers .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter((importer: any): boolean => - instance !== null && (!instance?.dimension || - importer.dimension === instance.dimension - )) + .filter( + (importer: any): boolean => + instance !== null && + (!instance?.dimension || importer.dimension === instance.dimension), + ) .map( (importer: any): JSX.Element => { const pending = !!projects[instance.id]; diff --git a/cvat-ui/src/components/import-dataset-modal/styles.scss b/cvat-ui/src/components/import-dataset-modal/styles.scss new file mode 100644 index 000000000000..4b8d86a4c939 --- /dev/null +++ b/cvat-ui/src/components/import-dataset-modal/styles.scss @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +@import '../../base.scss'; + +.cvat-modal-import-option-item > .ant-select-item-option-content, +.cvat-modal-import-select .ant-select-selection-item { + > span[role='img'] { + color: $info-icon-color; + margin-right: $grid-unit-size; + } +} + +.cvat-modal-import-header-question-icon { + margin-left: $grid-unit-size; + color: $text-color-secondary; +} From 3ed179434d420e457bf68513228fcc55b2eb23df Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 28 Oct 2021 13:56:16 +0300 Subject: [PATCH 22/57] Fixed linters problem --- cvat-ui/src/components/projects-page/actions-menu.tsx | 5 +++-- cvat/apps/dataset_manager/formats/cvat.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index b68e18ca5790..2bb27b3ba24b 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -7,7 +7,6 @@ import { useDispatch } from 'react-redux'; import Modal from 'antd/lib/modal'; import Menu from 'antd/lib/menu'; -import { MenuItem } from 'rc-menu'; import { deleteProjectAsync } from 'actions/projects-actions'; import { exportActions } from 'actions/export-actions'; import { importActions } from 'actions/import-actions'; @@ -42,7 +41,9 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { dispatch(exportActions.openExportModal(projectInstance))}> Export dataset - dispatch(importActions.openImportModal(projectInstance))}>Import dataset + dispatch(importActions.openImportModal(projectInstance))}> + Import dataset +
Delete diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index ca3eff18c042..a8b5e8127432 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -61,9 +61,9 @@ def __iter__(self): def __len__(self): return len(self._items) - def get(self, id, subset=DEFAULT_SUBSET_NAME): + def get(self, _id, subset=DEFAULT_SUBSET_NAME): assert subset in self._subsets, '{} not in {}'.format(subset, ', '.join(self._subsets)) - return super().get(id, subset) + return super().get(_id, subset) @staticmethod def _get_subsets_from_anno(path): @@ -820,7 +820,6 @@ def dump_track(idx, track): dumper.close_root() def load_anno(file_object, annotations): - from defusedxml import ElementTree supported_shapes = ('box', 'polygon', 'polyline', 'points', 'cuboid') context = ElementTree.iterparse(file_object, events=("start", "end")) context = iter(context) @@ -1013,4 +1012,4 @@ def _import(src_file, instance_data, load_data_callback=None): load_data_callback(dataset, instance_data) import_dm_annotations(dataset, instance_data) else: - load_task(src_file, instance_data) + load_anno(src_file, instance_data) From 04749348cd60d21eb8cd3872d24a1ec77415cccb Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 1 Nov 2021 12:26:36 +0300 Subject: [PATCH 23/57] Added import progress modal --- cvat-core/src/annotations.js | 7 +- cvat-core/src/project-implementation.js | 8 +- cvat-core/src/project.js | 3 +- cvat-core/src/server-proxy.js | 5 +- cvat-ui/src/actions/import-actions.ts | 6 +- .../import-dataset-modal.tsx | 163 +++++++++--------- .../import-dataset-status-modal.tsx | 34 ++++ .../import-dataset-modal/styles.scss | 14 ++ cvat-ui/src/reducers/import-reducer.ts | 10 ++ cvat-ui/src/reducers/interfaces.ts | 2 + cvat/apps/dataset_manager/bindings.py | 8 +- cvat/apps/dataset_manager/formats/cvat.py | 40 ++++- cvat/apps/engine/views.py | 26 ++- 13 files changed, 228 insertions(+), 98 deletions(-) create mode 100644 cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 246dd151cbff..c00dd5dba817 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -284,15 +284,18 @@ return result; } - async function importDataset(instance, format, file) { + async function importDataset(instance, format, file, updateStatusCallback) { if (!(format instanceof String || typeof format === 'string')) { throw new ArgumentError('Format must be a string'); } if (!(instance instanceof Project)) { throw new ArgumentError('Instance should ne a Project isntance'); } + if (!(typeof updateStatusCallback === 'function' || updateStatusCallback === null)) { + throw new ArgumentError('Callback should ne a function or null'); + } // TODO: check file - return serverProxy.projects.importDataset(instance.id, format, file); + return serverProxy.projects.importDataset(instance.id, format, file, updateStatusCallback); } function undoActions(session, count) { diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js index cb2961ed7ea2..d1d5947542ff 100644 --- a/cvat-core/src/project-implementation.js +++ b/cvat-core/src/project-implementation.js @@ -68,8 +68,12 @@ const result = exportDataset(this, format, customName, saveImages); return result; }; - projectClass.prototype.annotations.importDataset.implementation = async function (format, file) { - return importDataset(this, format, file); + projectClass.prototype.annotations.importDataset.implementation = async function ( + format, + file, + updateStatusCallback, + ) { + return importDataset(this, format, file, updateStatusCallback); }; return projectClass; diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index 5f9e625cce6f..42d625f85ef5 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -337,12 +337,13 @@ ); return result; }, - async importDataset(format, file) { + async importDataset(format, file, updateStatusCallback = null) { const result = await PluginRegistry.apiWrapper.call( this, Project.prototype.annotations.importDataset, format, file, + updateStatusCallback, ); return result; }, diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 86de6fc1d495..63aef9d0b68d 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -499,7 +499,7 @@ }; } - async function importDataset(id, format, file) { + async function importDataset(id, format, file, callback) { const { backendAPI } = config; const url = `${backendAPI}/projects/${id}/dataset`; @@ -513,6 +513,7 @@ proxy: config.proxy, }); if (response.status === 202) { + if (typeof callback === 'function') callback(response.data); setTimeout(request, 3000); } else if (response.status === 201) { resolve(); @@ -525,7 +526,7 @@ await Axios.post(`${url}?format=${format}`, formData, { proxy: config.proxy, }); - setTimeout(request, 3000); + setTimeout(request, 2000); } catch (_error) { reject(generateError(error)); } diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index f650e5aaa1c2..563b21c40241 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -13,6 +13,7 @@ export enum ImportActionTypes { IMPORT_DATASET = 'IMPORT_DATASET', IMPORT_DATASET_SUCCESS = 'IMPORT_DATASET_SUCCESS', IMPORT_DATASET_FAILED = 'IMPORT_DATASET_FAILED', + IMPORT_DATASET_UPDATE_STATUS = 'IMPORT_DATASET_UPDATE_STATUS', } export const importActions = { @@ -27,6 +28,8 @@ export const importActions = { instance, error, }), + importDatasetUpdateStatus: (progress: number, status: string) => + createAction(ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS, { progress, status }), }; export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch) => { @@ -37,7 +40,8 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T throw Error('Only one importing of dataset allowed at the same time'); } dispatch(importActions.importDataset(instance, format)); - await instance.annotations.importDataset(format, file); + await instance.annotations.importDataset(format, file, (response: any) => + dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message))); } catch (error) { dispatch(importActions.importDatasetFailed(instance, error)); return; diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index a15395a09350..cc5149b22044 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -21,6 +21,8 @@ import CVATTooltip from 'components/common/cvat-tooltip'; import { CombinedState } from 'reducers/interfaces'; import { importActions, importDatasetAsync } from 'actions/import-actions'; +import ImportDatasetStatusModal from './import-dataset-status-modal'; + type FormValues = { selectedFormat: string | undefined; }; @@ -60,87 +62,90 @@ function ImportDatasetModal(): JSX.Element { ); return ( - - Import dataset to project - - - - - )} - visible={modalVisible} - onCancel={closeModal} - onOk={() => form.submit()} - className='cvat-modal-import-project' - > -
+ + Import dataset to project + + + + + )} + visible={modalVisible} + onCancel={closeModal} + onOk={() => form.submit()} + className='cvat-modal-import-project' > - - - - { - if (!['application/zip', 'application/x-zip-compressed'].includes(_file.type)) { - message.error('Only ZIP archive is supported'); - } else { - setFile(_file); - } - return false; - }} - onRemove={() => { - setFile(null); - }} + -

- -

-

Click or drag file to this area

- - -
+ + + + { + if (!['application/zip', 'application/x-zip-compressed'].includes(_file.type)) { + message.error('Only ZIP archive is supported'); + } else { + setFile(_file); + } + return false; + }} + onRemove={() => { + setFile(null); + }} + > +

+ +

+

Click or drag file to this area

+ + + + + ); } diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx new file mode 100644 index 000000000000..a368ec48cd2f --- /dev/null +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx @@ -0,0 +1,34 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import './styles.scss'; +import React from 'react'; +import { useSelector } from 'react-redux'; +import Modal from 'antd/lib/modal'; +import Alert from 'antd/lib/alert'; +import Progress from 'antd/lib/progress'; + +import { CombinedState } from 'reducers/interfaces'; + +function ImportDatasetStatusModal(): JSX.Element { + const projects = useSelector((state: CombinedState) => state.import.projects); + const progress = useSelector((state: CombinedState) => state.import.progress); + const status = useSelector((state: CombinedState) => state.import.status); + const id = Object.keys(projects).length && Object.keys(projects)[0]; + + return ( + + + + + ); +} + +export default ImportDatasetStatusModal; diff --git a/cvat-ui/src/components/import-dataset-modal/styles.scss b/cvat-ui/src/components/import-dataset-modal/styles.scss index 4b8d86a4c939..a9d02ac02388 100644 --- a/cvat-ui/src/components/import-dataset-modal/styles.scss +++ b/cvat-ui/src/components/import-dataset-modal/styles.scss @@ -16,3 +16,17 @@ margin-left: $grid-unit-size; color: $text-color-secondary; } + +.cvat-modal-import-project-status .ant-modal-body { + display: flex; + align-items: center; + flex-flow: column; + + .ant-progress { + margin-bottom: $grid-unit-size * 2; + } + + .ant-alert { + width: 100%; + } +} diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index bab7723837ec..9a663fa2843a 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -9,6 +9,8 @@ import { ImportState } from './interfaces'; const defaultState: ImportState = { projects: {}, + progress: 0.0, + status: '', instance: null, modalVisible: false, }; @@ -39,6 +41,14 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor projects: activities, }; } + case ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS: { + const { progress, status } = action.payload; + return { + ...state, + progress, + status, + }; + } case ImportActionTypes.IMPORT_DATASET_FAILED: case ImportActionTypes.IMPORT_DATASET_SUCCESS: { const { instance } = action.payload; diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index e4642a761186..1e00f58a52d0 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -119,6 +119,8 @@ export interface ImportState { projects: { [pid: number]: string; }; + progress: number; + status: string; instance: any; modalVisible: boolean; } diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 218238b5a9b3..f87d5a0a1ff8 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: MIT import sys +import rq import os.path as osp from collections import namedtuple from typing import Any, Callable, DefaultDict, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union, Set @@ -1423,7 +1424,12 @@ def load_dataset_data(project_annotation, dataset: Dataset, project_data): for label in dataset.categories()[datumaro.AnnotationType.label].items: if not project_annotation.db_project.label_set.filter(name=label.name).exists(): raise CvatImportError(f'Target project does not have label with name "{label.name}"') - for subset in dataset.subsets().values(): + for subset_id, subset in enumerate(dataset.subsets().values()): + job = rq.get_current_job() + job.meta['status'] = 'Task from dataset is being created...' + job.meta['progress'] = subset_id / len(dataset.subsets().keys()) + job.save_meta() + task_fields = { 'project': project_annotation.db_project, 'name': subset.name, diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index a8b5e8127432..633e9e01d9f3 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -5,6 +5,8 @@ from io import BufferedWriter import os import os.path as osp +from glob import glob +from posixpath import basename from typing import Callable import zipfile from collections import OrderedDict @@ -48,8 +50,9 @@ def __init__(self, path, subsets=None): self._subsets = subsets super().__init__(subsets=self._subsets) + image_items = self._parse_images(images_dir, self._subsets) items, categories = self._parse(path) - self._items = list(self._load_items(items).values()) + self._items = list(self._load_items(items, image_items).values()) self._categories = categories def categories(self): @@ -81,6 +84,25 @@ def _get_subsets_from_anno(path): el.clear() return [DEFAULT_SUBSET_NAME] + @staticmethod + def _parse_images(image_dir, subsets): + items = OrderedDict() + + if subsets == [DEFAULT_SUBSET_NAME] and not osp.isdir(osp.join(image_dir, DEFAULT_SUBSET_NAME)): + for file in sorted(glob(osp.join(image_dir, '*.PNG')), key=osp.basename): + name = osp.splitext(osp.basename(file))[0] + items[(None, name)] = DatasetItem(id=name, annotations=[], + image=Image(path=file), subset=DEFAULT_SUBSET_NAME, + ) + else: + for subset in subsets: + for file in sorted(glob(osp.join(image_dir, subset, '*.PNG')), key=osp.basename): + name = osp.splitext(osp.basename(file))[0] + items[(subset, name)] = DatasetItem(id=name, annotations=[], + image=Image(path=file), subset=subset, + ) + return items + @classmethod def _parse(cls, path): context = ElementTree.iterparse(path, events=("start", "end")) @@ -369,20 +391,20 @@ def _parse_tag_ann(cls, ann, categories): attributes = ann.get('attributes') return Label(label_id, attributes=attributes, group=group) - def _load_items(self, parsed): + def _load_items(self, parsed, image_items): for (subset, frame_id), item_desc in parsed.items(): name = item_desc.get('name', 'frame_%06d.PNG' % int(frame_id)) image = osp.join(self._images_dir, subset, name) if subset else osp.join(self._images_dir, name) image_size = (item_desc.get('height'), item_desc.get('width')) if all(image_size): image = Image(path=image, size=tuple(map(int, image_size))) - - parsed[(subset, frame_id)] = DatasetItem(id=osp.splitext(name)[0], - subset=subset or DEFAULT_SUBSET_NAME, image=image, - annotations=item_desc.get('annotations'), - attributes={'frame': int(frame_id)}) - return parsed - + di = image_items.get((subset, osp.splitext(name)[0])) + di.subset = subset or DEFAULT_SUBSET_NAME + di.annotations = item_desc.get('annotations') + di.attributes = {'frame': int(frame_id)} + di.image = image if isinstance(image, Image) else di.image + image_items[(subset, osp.splitext(name)[0])] = di + return image_items dm_env.extractors.register('cvat', CvatExtractor) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a14f5da64a19..468aef3989cc 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -370,7 +370,10 @@ def dataset(self, request, pk): status=status.HTTP_500_INTERNAL_SERVER_ERROR ) else: - return Response(status=status.HTTP_202_ACCEPTED) + return Response( + data=self._get_rq_response('default', f'/api/v1/project/{pk}/dataset_import'), + status=status.HTTP_202_ACCEPTED + ) else: format_name = request.query_params.get("format", "") return _export_annotations( @@ -420,6 +423,25 @@ def annotations(self, request, pk): else: return Response("Format is not specified",status=status.HTTP_400_BAD_REQUEST) + @staticmethod + def _get_rq_response(queue, job_id): + queue = django_rq.get_queue(queue) + job = queue.fetch_job(job_id) + response = {} + if job is None or job.is_finished: + response = { "state": "Finished" } + elif job.is_queued: + response = { "state": "Queued" } + elif job.is_failed: + response = { "state": "Failed", "message": job.exc_info } + else: + response = { "state": "Started" } + response['message'] = job.meta.get('status', '') + response['progress'] = job.meta.get('progress', 0.) + + + return response + class TaskFilter(filters.FilterSet): project = filters.CharFilter(field_name="project__name", lookup_expr="icontains") name = filters.CharFilter(field_name="name", lookup_expr="icontains") @@ -1698,6 +1720,8 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name): ) rq_job.meta['tmp_file'] = filename rq_job.meta['tmp_file_descriptor'] = fd + rq_job.meta['status'] = 'Dataset import has been started...' + rq_job.meta['progress'] = 0. rq_job.save_meta() else: #TODO: Should it be a response? From 29204ba8cfe863ed6e20185bf0b4e5a80f7f4de9 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 1 Nov 2021 12:31:43 +0300 Subject: [PATCH 24/57] Fixed menu items --- cvat-ui/src/components/projects-page/actions-menu.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index 2bb27b3ba24b..cda7771a314d 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -38,14 +38,16 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( - dispatch(exportActions.openExportModal(projectInstance))}> + dispatch(exportActions.openExportModal(projectInstance))}> Export dataset - dispatch(importActions.openImportModal(projectInstance))}> + dispatch(importActions.openImportModal(projectInstance))}> Import dataset
- Delete + + Delete + ); } From da769f3beeb64359d739abde1f9c67e55bd927f5 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 02:33:06 +0300 Subject: [PATCH 25/57] Fixed linters --- cvat-ui/src/actions/import-actions.ts | 25 +++++++++++-------- .../import-dataset-modal.tsx | 5 ++-- cvat/apps/dataset_manager/formats/cvat.py | 1 - 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 563b21c40241..38ed11dfb823 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -19,17 +19,21 @@ export enum ImportActionTypes { export const importActions = { openImportModal: (instance: any) => createAction(ImportActionTypes.OPEN_IMPORT_MODAL, { instance }), closeImportModal: () => createAction(ImportActionTypes.CLOSE_IMPORT_MODAL), - importDataset: (instance: any, format: string) => - createAction(ImportActionTypes.IMPORT_DATASET, { instance, format }), - importDatasetSuccess: (instance: any, format: string) => - createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS, { instance, format }), - importDatasetFailed: (instance: any, error: any) => + importDataset: (instance: any, format: string) => ( + createAction(ImportActionTypes.IMPORT_DATASET, { instance, format }) + ), + importDatasetSuccess: (instance: any, format: string) => ( + createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS, { instance, format }) + ), + importDatasetFailed: (instance: any, error: any) => ( createAction(ImportActionTypes.IMPORT_DATASET_FAILED, { instance, error, - }), - importDatasetUpdateStatus: (progress: number, status: string) => - createAction(ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS, { progress, status }), + }) + ), + importDatasetUpdateStatus: (progress: number, status: string) => ( + createAction(ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS, { progress, status }) + ), }; export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch) => { @@ -40,8 +44,9 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T throw Error('Only one importing of dataset allowed at the same time'); } dispatch(importActions.importDataset(instance, format)); - await instance.annotations.importDataset(format, file, (response: any) => - dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message))); + await instance.annotations.importDataset(format, file, (response: any) => ( + dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message)) + )); } catch (error) { dispatch(importActions.importDatasetFailed(instance, error)); return; diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index cc5149b22044..d38187b0b492 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -98,9 +98,10 @@ function ImportDatasetModal(): JSX.Element { {importers .sort((a: any, b: any) => a.name.localeCompare(b.name)) .filter( - (importer: any): boolean => + (importer: any): boolean => ( instance !== null && - (!instance?.dimension || importer.dimension === instance.dimension), + (!instance?.dimension || importer.dimension === instance.dimension) + ), ) .map( (importer: any): JSX.Element => { diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 633e9e01d9f3..95e55bcea0a0 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -6,7 +6,6 @@ import os import os.path as osp from glob import glob -from posixpath import basename from typing import Callable import zipfile from collections import OrderedDict From 9a0c1ac20ccff488d60b08d4c2f910cfbe9f6a66 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 02:48:16 +0300 Subject: [PATCH 26/57] Fixed imports --- cvat/apps/dataset_manager/formats/cvat.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 95e55bcea0a0..1ad61450f59e 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -12,11 +12,12 @@ from tempfile import TemporaryDirectory from defusedxml import ElementTree -from datumaro.components.dataset import Dataset -from datumaro.components.extractor import ( - AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points, - Polygon, PolyLine, Extractor, DEFAULT_SUBSET_NAME +from datumaro.components.dataset import Dataset, DatasetItem +from datumaro.components.extractor import Importer, Extractor, DEFAULT_SUBSET_NAME +from datumaro.components.annotation import ( + AnnotationType, Bbox, Points, Polygon, PolyLine, Label, LabelCategories, ) + from datumaro.util.image import Image from cvat.apps.dataset_manager.bindings import TaskData, match_dm_item, ProjectData, get_defaulted_subset, import_dm_annotations From ea8516fe0fa106e9ba835f8ab03c4e83db985833 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 11:14:45 +0300 Subject: [PATCH 27/57] Fixed TODOs --- cvat-core/src/annotations.js | 4 +++- cvat-ui/src/actions/import-actions.ts | 1 - cvat/apps/dataset_manager/project.py | 6 ++++++ cvat/apps/engine/serializers.py | 5 +++++ cvat/apps/engine/views.py | 7 +------ 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index c00dd5dba817..495bdabbd783 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -294,7 +294,9 @@ if (!(typeof updateStatusCallback === 'function' || updateStatusCallback === null)) { throw new ArgumentError('Callback should ne a function or null'); } - // TODO: check file + if (!(file instanceof File && file.name.split('.').reverse()[0])) { + throw new ArgumentError('File should be file instance with ZIP extension'); + } return serverProxy.projects.importDataset(instance.id, format, file, updateStatusCallback); } diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 38ed11dfb823..92886b465efc 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -53,7 +53,6 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T } dispatch(importActions.importDatasetSuccess(instance, format)); - // TODO: should we call this in component or here? dispatch(getProjectsAsync({ id: instance.id })); }; diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py index 897e7f098102..a649ba2205c9 100644 --- a/cvat/apps/dataset_manager/project.py +++ b/cvat/apps/dataset_manager/project.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +import rq from typing import Any, Callable, List, Mapping, Tuple from django.db import transaction @@ -150,6 +151,11 @@ def data(self) -> dict: @transaction.atomic def import_dataset_as_project(project_id, dataset_file, format_name): + rq_job = rq.get_current_job() + rq_job.meta['status'] = 'Dataset import has been started...' + rq_job.meta['progress'] = 0. + rq_job.save_meta() + project = ProjectAnnotationAndData(project_id) project.init_from_db() diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index a39c8bb96103..beb2aa39c1bd 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -720,6 +720,11 @@ class AnnotationFileSerializer(serializers.Serializer): class DatasetFileSerializer(serializers.Serializer): dataset_file = serializers.FileField() + def validate_dataset_file(value): + if os.path.splitext(value.name)[1] != '.zip': + raise serializers.ValidationError('Dataset file should be zip archive') + return value + class TaskFileSerializer(serializers.Serializer): task_file = serializers.FileField() diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 468aef3989cc..2a44e9eb6cc0 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -364,7 +364,6 @@ def dataset(self, request, pk): os.close(rq_job.meta['tmp_file_descriptor']) os.remove(rq_job.meta['tmp_file']) rq_job.delete() - #TODO: Should we check CVATImportError here? return Response( data=str(rq_job.exc_info), status=status.HTTP_500_INTERNAL_SERVER_ERROR @@ -1712,7 +1711,6 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name): for chunk in dataset_file.chunks(): f.write(chunk) - #TODO: should we check zip here? rq_job = queue.enqueue_call( func=rq_func, args=(pk, filename, format_name), @@ -1720,11 +1718,8 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name): ) rq_job.meta['tmp_file'] = filename rq_job.meta['tmp_file_descriptor'] = fd - rq_job.meta['status'] = 'Dataset import has been started...' - rq_job.meta['progress'] = 0. rq_job.save_meta() else: - #TODO: Should it be a response? - raise ValidationError("Import job already exists") + return Response(status=status.HTTP_409_CONFLICT, data='Import job already exists') return Response(status=status.HTTP_202_ACCEPTED) From 1b6af2685d1f7bc031e95c1f0234736ffc03068f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 11:23:24 +0300 Subject: [PATCH 28/57] Update cvat/apps/dataset_manager/bindings.py Co-authored-by: Maxim Zhiltsov --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 69d78dc96b9a..45fad02aa4c9 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1400,7 +1400,7 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr occluded=ann.attributes.pop('occluded', None) == True, z_order=ann.z_order, group=group_map.get(ann.group, 0), - source='manual', + source=str(attributes.pop('source')).lower() if str(attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', attributes=[instance_data.Attribute(name=n, value=str(v)) for n, v in ann.attributes.items()], )) From e30e5e774ac170785f9d92adc9cd8e626fef3ac8 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 11:23:40 +0300 Subject: [PATCH 29/57] Update cvat/apps/dataset_manager/bindings.py Co-authored-by: Maxim Zhiltsov --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 45fad02aa4c9..f33020066794 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1415,7 +1415,7 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr )) except Exception as e: raise CvatImportError("Image {}: can't import annotation " - "#{} ({}): {}".format(item.id, idx, ann.type.name, e)) + "#{} ({}): {}".format(item.id, idx, ann.type.name, e)) from e def import_labels_to_project(project_annotation, dataset: Dataset): labels = [] From 307b557b1e6ad94a1274ed457ef11f3412414168 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 2 Nov 2021 12:36:58 +0300 Subject: [PATCH 30/57] Fixed comments --- cvat/apps/dataset_manager/bindings.py | 35 +++++++-------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index f33020066794..b022950551ab 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -27,6 +27,8 @@ from .annotation import AnnotationIR, AnnotationManager, TrackManager +CVAT_INTERNAL_ATTRIBUTES = {'occluded', 'outside', 'keyframe', 'track_id'} + class InstanceLabelData: Attribute = NamedTuple('Attribute', [('name', str), ('value', Any)]) @@ -35,6 +37,7 @@ def __init__(self, instance: Union[Task, Project]) -> None: db_labels = instance.label_set.all().prefetch_related('attributespec_set').order_by('pk') + # If this flag is set to true, create attribute within anntations import self._soft_attribute_import = False self._label_mapping = OrderedDict[int, Label]( ((db_label.id, db_label) for db_label in db_labels), @@ -448,7 +451,7 @@ def _import_tag(self, tag): _tag['attributes'] = [self._import_attribute(label_id, attrib) for attrib in _tag['attributes'] if self._get_attribute_id(label_id, attrib.name) or ( - self.soft_attribute_import and attrib.name not in {'occluded', 'outside', 'keyframe', 'track_id'} + self.soft_attribute_import and attrib.name not in CVAT_INTERNAL_ATTRIBUTES ) ] return _tag @@ -461,7 +464,7 @@ def _import_shape(self, shape): _shape['attributes'] = [self._import_attribute(label_id, attrib) for attrib in _shape['attributes'] if self._get_attribute_id(label_id, attrib.name) or ( - self.soft_attribute_import and attrib.name not in {'occluded', 'outside', 'keyframe', 'track_id'} + self.soft_attribute_import and attrib.name not in CVAT_INTERNAL_ATTRIBUTES ) ] _shape['points'] = list(map(float, _shape['points'])) @@ -480,13 +483,13 @@ def _import_track(self, track): _track['attributes'] = [self._import_attribute(label_id, attrib) for attrib in shape['attributes'] if self._get_immutable_attribute_id(label_id, attrib.name) or ( - self.soft_attribute_import and attrib.name not in {'occluded', 'outside', 'keyframe', 'track_id'} + self.soft_attribute_import and attrib.name not in CVAT_INTERNAL_ATTRIBUTES ) ] shape['attributes'] = [self._import_attribute(label_id, attrib, mutable=True) for attrib in shape['attributes'] if self._get_mutable_attribute_id(label_id, attrib.name) or ( - self.soft_attribute_import and attrib.name not in {'occluded', 'outside', 'keyframe', 'track_id'} + self.soft_attribute_import and attrib.name not in CVAT_INTERNAL_ATTRIBUTES ) ] shape['points'] = list(map(float, shape['points'])) @@ -565,27 +568,6 @@ def match_frame_fuzzy(self, path): return None class ProjectData(InstanceLabelData): - LabeledShape = NamedTuple('LabledShape', - [('type', str), ('frame', int), ('label', str), ('points', List[float]), ('occluded', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('task_id', int), ('subset', str)], - ) - LabeledShape.__new__.__defaults__ = ('manual', 0, 0, None, None) - TrackedShape = NamedTuple('TrackedShape', - [('type', str), ('frame', int), ('points', List[float]), ('occluded', bool), ('outside', bool), ('keyframe', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('label', str), ('track_id', int)], - ) - TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) - Track = NamedTuple('Track', - [('label', str), ('shapes', List[TrackedShape]), ('source', str), ('group', int), ('task_id', int), ('subset', str)], - ) - Track.__new__.__defaults__ = ('manual', 0, None, None) - Tag = NamedTuple('Tag', - [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int), ('subset', str)], - ) - Tag.__new__.__defaults__ = ('manual', 0, None, None) - Frame = NamedTuple('Frame', - [('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag]), ('task_id', int), ('subset', str)], - ) - Frame.__new__.__defaults__ = (None, None) - def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str = '', task_annotations: Mapping[int, Any] = None, project_annotation=None): self._annotation_irs = annotation_irs self._db_project = db_project @@ -1400,7 +1382,8 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr occluded=ann.attributes.pop('occluded', None) == True, z_order=ann.z_order, group=group_map.get(ann.group, 0), - source=str(attributes.pop('source')).lower() if str(attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', + source=str(ann.attributes.pop('source')).lower() \ + if str(ann.attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', attributes=[instance_data.Attribute(name=n, value=str(v)) for n, v in ann.attributes.items()], )) From 7958c00652a808af191ca9e45529e78cf912e3d1 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 10 Nov 2021 12:09:06 +0300 Subject: [PATCH 31/57] Fixed tests issues with CVAT format --- cvat/apps/dataset_manager/bindings.py | 126 +++++++++++++++++++--- cvat/apps/dataset_manager/formats/cvat.py | 43 +++++--- cvat/apps/engine/serializers.py | 1 + 3 files changed, 140 insertions(+), 30 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index b022950551ab..c2f4863e2b55 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -6,6 +6,7 @@ import sys import rq import os.path as osp +from attr import attrib, attrs from collections import namedtuple from pathlib import Path from typing import (Any, Callable, DefaultDict, Dict, List, Literal, Mapping, @@ -568,6 +569,68 @@ def match_frame_fuzzy(self, path): return None class ProjectData(InstanceLabelData): + @attrs + class LabeledShape: + type: str = attrib() + frame: int = attrib() + label: str = attrib() + points: List[float] = attrib() + occluded: bool = attrib() + attributes: List[InstanceLabelData.Attribute] = attrib() + source: str = attrib() + source: str = attrib(default='manual') + group: int = attrib(default=0) + z_order: int = attrib(default=0) + task_id: int = attrib(default=None) + subset: str = attrib(default=None) + + @attrs + class TrackedShape: + type: str = attrib() + frame: int = attrib() + points: List[float] = attrib() + occluded: bool = attrib() + outside: bool = attrib() + keyframe: bool = attrib() + attributes: List[InstanceLabelData.Attribute] = attrib() + source: str = attrib(default='manual') + group: int = attrib(default=0) + z_order: int = attrib(default=0) + label: str = attrib(default=None) + track_id: int = attrib(default=0) + + @attrs + class Track: + label: str = attrib() + shapes: List['ProjectData.TrackedShape'] = attrib() + source: str = attrib(default='manual') + group: int = attrib(default=0) + task_id: int = attrib(default=None) + subset: str = attrib(default=None) + + @attrs + class Tag: + frame: int = attrib() + label: str = attrib() + attributes: List[InstanceLabelData.Attribute] = attrib() + source: str = attrib(default='manual') + group: int = attrib(default=0) + task_id: int = attrib(default=None) + subset: str = attrib(default=None) + + @attrs + class Frame: + idx: int = attrib() + id: int = attrib() + frame: int = attrib() + name: str = attrib() + width: int = attrib() + height: int = attrib() + labeled_shapes: List[Union['ProjectData.LabeledShape', 'ProjectData.TrackedShape']] = attrib() + tags: List['ProjectData.Tag'] = attrib() + task_id: int = attrib(default=None) + subset: str = attrib(default=None) + def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str = '', task_annotations: Mapping[int, Any] = None, project_annotation=None): self._annotation_irs = annotation_irs self._db_project = db_project @@ -1343,6 +1406,8 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr root_hint = find_dataset_root(dm_dataset, instance_data) + tracks = {} + for item in dm_dataset: frame_number = instance_data.abs_frame_id( match_dm_item(item, instance_data, root_hint=root_hint)) @@ -1374,19 +1439,50 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr except Exception as e: ann.points = ann.points ann.z_order = 0 - instance_data.add_shape(instance_data.LabeledShape( - type=shapes[ann.type], - frame=frame_number, - points=ann.points, - label=label_cat.items[ann.label].name, - occluded=ann.attributes.pop('occluded', None) == True, - z_order=ann.z_order, - group=group_map.get(ann.group, 0), - source=str(ann.attributes.pop('source')).lower() \ - if str(ann.attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', - attributes=[instance_data.Attribute(name=n, value=str(v)) - for n, v in ann.attributes.items()], - )) + + track_id = ann.attributes.pop('track_id', None) + if track_id is None or dm_dataset.format != 'cvat' : + instance_data.add_shape(instance_data.LabeledShape( + type=shapes[ann.type], + frame=frame_number, + points=ann.points, + label=label_cat.items[ann.label].name, + occluded=ann.attributes.pop('occluded', None) == True, + z_order=ann.z_order, + group=group_map.get(ann.group, 0), + source=str(ann.attributes.pop('source')).lower() \ + if str(ann.attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', + attributes=[instance_data.Attribute(name=n, value=str(v)) + for n, v in ann.attributes.items()], + )) + continue + + if ann.attributes.get('keyframe', None) == True or ann.attributes.get('outside', None) == True: + track = instance_data.TrackedShape( + type=shapes[ann.type], + frame=frame_number, + occluded=ann.attributes.pop('occluded', None) == True, + outside=ann.attributes.pop('outside', None) == True, + keyframe=ann.attributes.get('keyframe', None) == True, + points=ann.points, + z_order=ann.z_order, + source=str(ann.attributes.pop('source')).lower() \ + if str(ann.attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', + attributes=[instance_data.Attribute(name=n, value=str(v)) + for n, v in ann.attributes.items()], + ) + + if track_id not in tracks: + tracks[track_id] = instance_data.Track( + label=label_cat.items[ann.label].name, + group=group_map.get(ann.group, 0), + source=str(ann.attributes.pop('source')).lower() \ + if str(ann.attributes.get('source', None)).lower() in {'auto', 'manual'} else 'manual', + shapes=[], + ) + + tracks[track_id].shapes.append(track) + elif ann.type == datum_annotation.AnnotationType.label: instance_data.add_tag(instance_data.Tag( frame=frame_number, @@ -1400,6 +1496,10 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr raise CvatImportError("Image {}: can't import annotation " "#{} ({}): {}".format(item.id, idx, ann.type.name, e)) from e + for track in tracks.values(): + instance_data.add_track(track) + + def import_labels_to_project(project_annotation, dataset: Dataset): labels = [] label_names = [] diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 1ad61450f59e..82cb4cabb612 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -29,7 +29,7 @@ class CvatPath: IMAGES_DIR = 'images' - IMAGE_EXT = '.jpg' + MEDIA_EXTS = ('.jpg', '.jpeg', '.png') BUILTIN_ATTRS = {'occluded', 'outside', 'keyframe', 'track_id'} @@ -89,18 +89,20 @@ def _parse_images(image_dir, subsets): items = OrderedDict() if subsets == [DEFAULT_SUBSET_NAME] and not osp.isdir(osp.join(image_dir, DEFAULT_SUBSET_NAME)): - for file in sorted(glob(osp.join(image_dir, '*.PNG')), key=osp.basename): - name = osp.splitext(osp.basename(file))[0] - items[(None, name)] = DatasetItem(id=name, annotations=[], - image=Image(path=file), subset=DEFAULT_SUBSET_NAME, - ) + for file in sorted(glob(osp.join(image_dir, '*.*')), key=osp.basename): + name, ext = osp.splitext(osp.basename(file)) + if ext.lower() in CvatPath.MEDIA_EXTS: + items[(None, name)] = DatasetItem(id=name, annotations=[], + image=Image(path=file), subset=DEFAULT_SUBSET_NAME, + ) else: for subset in subsets: - for file in sorted(glob(osp.join(image_dir, subset, '*.PNG')), key=osp.basename): - name = osp.splitext(osp.basename(file))[0] - items[(subset, name)] = DatasetItem(id=name, annotations=[], - image=Image(path=file), subset=subset, - ) + for file in sorted(glob(osp.join(image_dir, subset, '*.*')), key=osp.basename): + name, ext = osp.splitext(osp.basename(file)) + if ext.lower() in CvatPath.MEDIA_EXTS: + items[(subset, name)] = DatasetItem(id=name, annotations=[], + image=Image(path=file), subset=subset, + ) return items @classmethod @@ -164,7 +166,7 @@ def _parse(cls, path): attr_type = attribute_types.get(el.attrib['name']) if el.text in ['true', 'false']: attr_value = attr_value == 'true' - elif attr_type is not None and attr_type != 'text': + elif attr_type is not None and attr_type != 'text': try: attr_value = float(attr_value) except ValueError: @@ -398,7 +400,9 @@ def _load_items(self, parsed, image_items): image_size = (item_desc.get('height'), item_desc.get('width')) if all(image_size): image = Image(path=image, size=tuple(map(int, image_size))) - di = image_items.get((subset, osp.splitext(name)[0])) + di = image_items.get((subset, osp.splitext(name)[0]), DatasetItem( + id=name, annotations=[], + )) di.subset = subset or DEFAULT_SUBSET_NAME di.annotations = item_desc.get('annotations') di.attributes = {'frame': int(frame_id)} @@ -1029,9 +1033,14 @@ def _import(src_file, instance_data, load_data_callback=None): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) - dataset = Dataset.import_from(tmp_dir, 'cvat', env=dm_env) - if load_data_callback is not None: - load_data_callback(dataset, instance_data) - import_dm_annotations(dataset, instance_data) + if isinstance(instance_data, ProjectData): + dataset = Dataset.import_from(tmp_dir, 'cvat', env=dm_env) + if load_data_callback is not None: + load_data_callback(dataset, instance_data) + import_dm_annotations(dataset, instance_data) + else: + anno_paths = glob(osp.join(tmp_dir, '**', '*.xml'), recursive=True) + for p in anno_paths: + load_anno(p, instance_data) else: load_anno(src_file, instance_data) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index beb2aa39c1bd..ab62735cecfd 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -720,6 +720,7 @@ class AnnotationFileSerializer(serializers.Serializer): class DatasetFileSerializer(serializers.Serializer): dataset_file = serializers.FileField() + @staticmethod def validate_dataset_file(value): if os.path.splitext(value.name)[1] != '.zip': raise serializers.ValidationError('Dataset file should be zip archive') From a4b02fb5d3f10084d3b175fcadeac80365f173f0 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 10 Nov 2021 13:35:52 +0300 Subject: [PATCH 32/57] Fixed cypress tests --- tests/cypress/support/commands_projects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cypress/support/commands_projects.js b/tests/cypress/support/commands_projects.js index 0261987c6a74..1349029100ac 100644 --- a/tests/cypress/support/commands_projects.js +++ b/tests/cypress/support/commands_projects.js @@ -70,7 +70,7 @@ Cypress.Commands.add('deleteProject', (projectName, projectID, expectedResult = Cypress.Commands.add('exportProject', ({ projectName, as, type, dumpType, archiveCustomeName }) => { cy.projectActions(projectName); cy.intercept('GET', `/api/v1/projects/**/${type}**`).as(as); - cy.get('.cvat-project-actions-menu').contains('Export project dataset').click(); + cy.get('.cvat-project-actions-menu').contains('Export dataset').click(); cy.get('.cvat-modal-export-project').should('be.visible').find('.cvat-modal-export-select').click(); cy.contains('.cvat-modal-export-option-item', dumpType).should('be.visible').click(); cy.get('.cvat-modal-export-select').should('contain.text', dumpType); From 199308b4e89b5b876e669e125f1e6a27a2b4bf58 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:14:58 +0300 Subject: [PATCH 33/57] Update cvat-core/src/annotations.js Co-authored-by: Boris Sekachev --- cvat-core/src/annotations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 495bdabbd783..fc9d1b30ec64 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -289,7 +289,7 @@ throw new ArgumentError('Format must be a string'); } if (!(instance instanceof Project)) { - throw new ArgumentError('Instance should ne a Project isntance'); + throw new ArgumentError('Instance should be a Project instance'); } if (!(typeof updateStatusCallback === 'function' || updateStatusCallback === null)) { throw new ArgumentError('Callback should ne a function or null'); From 65c47680ab90720e2dec87a4bf93a80198d30aad Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:15:06 +0300 Subject: [PATCH 34/57] Update cvat-core/src/annotations.js Co-authored-by: Boris Sekachev --- cvat-core/src/annotations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index fc9d1b30ec64..f3bd2706b3d6 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -292,7 +292,7 @@ throw new ArgumentError('Instance should be a Project instance'); } if (!(typeof updateStatusCallback === 'function' || updateStatusCallback === null)) { - throw new ArgumentError('Callback should ne a function or null'); + throw new ArgumentError('Callback should be a function or null'); } if (!(file instanceof File && file.name.split('.').reverse()[0])) { throw new ArgumentError('File should be file instance with ZIP extension'); From f1e9bf6a188c36021eac8794bea4f0796ef98536 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:15:37 +0300 Subject: [PATCH 35/57] Update cvat-ui/src/actions/import-actions.ts Co-authored-by: Boris Sekachev --- cvat-ui/src/actions/import-actions.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 92886b465efc..1a63be260749 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -36,10 +36,9 @@ export const importActions = { ), }; -export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch) => { +export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch, getState) => { try { - const store = getCVATStore(); - const state: CombinedState = store.getState(); + const state: CombinedState = getState(); if (state.import.projects[instance.id]) { throw Error('Only one importing of dataset allowed at the same time'); } From 91936925d90bdc1486ba710e5a8c5a91c6e230b1 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:15:55 +0300 Subject: [PATCH 36/57] Update cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx Co-authored-by: Boris Sekachev --- .../components/import-dataset-modal/import-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index d38187b0b492..a43728bbfb27 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -55,7 +55,7 @@ function ImportDatasetModal(): JSX.Element { Notification.info({ message: 'Dataset export started', description: `Dataset import was started for project #${instance?.id}. `, - className: 'cvat-notification-notice-import-project-start', + className: 'cvat-notification-notice-import-dataset-start', }); }, [instance?.id, file], From f754353e76a222b8e9e3c25ce96a7b4dc13c45ec Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:16:04 +0300 Subject: [PATCH 37/57] Update cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx Co-authored-by: Boris Sekachev --- .../components/import-dataset-modal/import-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index a43728bbfb27..656c74494594 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -112,7 +112,7 @@ function ImportDatasetModal(): JSX.Element { value={importer.name} key={importer.name} disabled={disabled} - className='cvat-modal-import-option-item' + className='cvat-modal-import-dataset-option-item' > {importer.name} From ba52e9da0678fa9e4b219c1b2e249c19a0984425 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:16:17 +0300 Subject: [PATCH 38/57] Update cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx Co-authored-by: Boris Sekachev --- .../components/import-dataset-modal/import-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index 656c74494594..56b3102ecdd1 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -81,7 +81,7 @@ function ImportDatasetModal(): JSX.Element { visible={modalVisible} onCancel={closeModal} onOk={() => form.submit()} - className='cvat-modal-import-project' + className='cvat-modal-import-dataset' >
Date: Tue, 16 Nov 2021 10:16:42 +0300 Subject: [PATCH 39/57] Update cvat-core/src/server-proxy.js Co-authored-by: Boris Sekachev --- cvat-core/src/server-proxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 3659d0885b3b..bd5d7e83331c 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -513,7 +513,7 @@ proxy: config.proxy, }); if (response.status === 202) { - if (typeof callback === 'function') callback(response.data); + if (callback) callback(response.data); setTimeout(request, 3000); } else if (response.status === 201) { resolve(); From 99d522a1c3ba5ce024e8e34d20d84fb1e5281940 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 10:17:21 +0300 Subject: [PATCH 40/57] Update cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx Co-authored-by: Boris Sekachev --- .../import-dataset-modal/import-dataset-status-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx index a368ec48cd2f..d8c3dd4886d1 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx @@ -19,7 +19,7 @@ function ImportDatasetStatusModal(): JSX.Element { return ( Date: Tue, 16 Nov 2021 10:17:38 +0300 Subject: [PATCH 41/57] Update cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx Co-authored-by: Boris Sekachev --- .../import-dataset-modal/import-dataset-status-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx index d8c3dd4886d1..2100d89d6b34 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-status-modal.tsx @@ -23,7 +23,7 @@ function ImportDatasetStatusModal(): JSX.Element { visible={!!id} closable={false} footer={null} - className='cvat-modal-import-project-status' + className='cvat-modal-import-dataset-status' > From 993b6f4772c6420b68325cbe69afb154d11fa7dd Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 20:15:12 +0300 Subject: [PATCH 42/57] Fixed comments --- cvat-core/src/annotations.js | 10 ++--- cvat-core/src/server-proxy.js | 25 +++++------ cvat-ui/src/actions/import-actions.ts | 43 ++++++++++--------- .../import-dataset-modal.tsx | 6 +-- .../import-dataset-status-modal.tsx | 9 ++-- .../import-dataset-modal/styles.scss | 2 +- cvat-ui/src/reducers/import-reducer.ts | 17 ++------ cvat-ui/src/reducers/interfaces.ts | 4 +- 8 files changed, 51 insertions(+), 65 deletions(-) diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index f3bd2706b3d6..e2868caf7031 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -284,17 +284,17 @@ return result; } - async function importDataset(instance, format, file, updateStatusCallback) { - if (!(format instanceof String || typeof format === 'string')) { + function importDataset(instance, format, file, updateStatusCallback = () => {}) { + if (!(typeof format === 'string')) { throw new ArgumentError('Format must be a string'); } if (!(instance instanceof Project)) { throw new ArgumentError('Instance should be a Project instance'); } - if (!(typeof updateStatusCallback === 'function' || updateStatusCallback === null)) { - throw new ArgumentError('Callback should be a function or null'); + if (!(typeof updateStatusCallback === 'function')) { + throw new ArgumentError('Callback should be a function'); } - if (!(file instanceof File && file.name.split('.').reverse()[0])) { + if (!(['application/zip', 'application/x-zip-compressed'].includes(file.type))) { throw new ArgumentError('File should be file instance with ZIP extension'); } return serverProxy.projects.importDataset(instance.id, format, file, updateStatusCallback); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index bd5d7e83331c..e3cc517cb5d9 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -507,36 +507,31 @@ formData.append('dataset_file', file); return new Promise((resolve, reject) => { - async function request() { + async function requestStatus() { try { const response = await Axios.get(`${url}?action=import_status`, { proxy: config.proxy, }); if (response.status === 202) { if (callback) callback(response.data); - setTimeout(request, 3000); + setTimeout(requestStatus, 3000); } else if (response.status === 201) { resolve(); } else { reject(generateError(response)); } } catch (error) { - if (error.response.status === 404) { - try { - await Axios.post(`${url}?format=${format}`, formData, { - proxy: config.proxy, - }); - setTimeout(request, 2000); - } catch (_error) { - reject(generateError(error)); - } - } else { - reject(generateError(error)); - } + reject(generateError(error)); } } - setTimeout(request); + Axios.post(`${url}?format=${format}`, formData, { + proxy: config.proxy, + }).then(() => { + setTimeout(requestStatus, 2000); + }).catch((error) => { + reject(generateError(error)); + }); }); } diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 1a63be260749..46fa40ec9242 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: MIT -import { getCVATStore } from 'cvat-store'; import { createAction, ActionUnion, ThunkAction } from 'utils/redux'; import { CombinedState } from 'reducers/interfaces'; import { getProjectsAsync } from './projects-actions'; @@ -19,11 +18,11 @@ export enum ImportActionTypes { export const importActions = { openImportModal: (instance: any) => createAction(ImportActionTypes.OPEN_IMPORT_MODAL, { instance }), closeImportModal: () => createAction(ImportActionTypes.CLOSE_IMPORT_MODAL), - importDataset: (instance: any, format: string) => ( - createAction(ImportActionTypes.IMPORT_DATASET, { instance, format }) + importDataset: (format: string) => ( + createAction(ImportActionTypes.IMPORT_DATASET, { format }) ), - importDatasetSuccess: (instance: any, format: string) => ( - createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS, { instance, format }) + importDatasetSuccess: () => ( + createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS) ), importDatasetFailed: (instance: any, error: any) => ( createAction(ImportActionTypes.IMPORT_DATASET_FAILED, { @@ -36,23 +35,25 @@ export const importActions = { ), }; -export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => async (dispatch, getState) => { - try { - const state: CombinedState = getState(); - if (state.import.projects[instance.id]) { - throw Error('Only one importing of dataset allowed at the same time'); +export const importDatasetAsync = (instance: any, format: string, file: File): ThunkAction => ( + async (dispatch, getState) => { + try { + const state: CombinedState = getState(); + if (state.import.format !== null) { + throw Error('Only one importing of dataset allowed at the same time'); + } + dispatch(importActions.importDataset(format)); + await instance.annotations.importDataset(format, file, (response: any) => ( + dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message)) + )); + } catch (error) { + dispatch(importActions.importDatasetFailed(instance, error)); + return; } - dispatch(importActions.importDataset(instance, format)); - await instance.annotations.importDataset(format, file, (response: any) => ( - dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message)) - )); - } catch (error) { - dispatch(importActions.importDatasetFailed(instance, error)); - return; - } - dispatch(importActions.importDatasetSuccess(instance, format)); - dispatch(getProjectsAsync({ id: instance.id })); -}; + dispatch(importActions.importDatasetSuccess()); + dispatch(getProjectsAsync({ id: instance.id })); + } +); export type ImportActions = ActionUnion; diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index 56b3102ecdd1..3367f8520b34 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -32,7 +32,7 @@ function ImportDatasetModal(): JSX.Element { const [file, setFile] = useState(null); const modalVisible = useSelector((state: CombinedState) => state.import.modalVisible); const instance = useSelector((state: CombinedState) => state.import.instance); - const projects = useSelector((state: CombinedState) => state.import.projects); + const format = useSelector((state: CombinedState) => state.import.format); const importers = useSelector((state: CombinedState) => state.formats.annotationFormats.loaders); const dispatch = useDispatch(); @@ -105,7 +105,7 @@ function ImportDatasetModal(): JSX.Element { ) .map( (importer: any): JSX.Element => { - const pending = !!projects[instance.id]; + const pending = format !== null; const disabled = !importer.enabled || pending; return ( state.import.projects); + const project = useSelector((state: CombinedState) => state.import.instance); + const format = useSelector((state: CombinedState) => state.import.format); const progress = useSelector((state: CombinedState) => state.import.progress); const status = useSelector((state: CombinedState) => state.import.status); - const id = Object.keys(projects).length && Object.keys(projects)[0]; return ( diff --git a/cvat-ui/src/components/import-dataset-modal/styles.scss b/cvat-ui/src/components/import-dataset-modal/styles.scss index a9d02ac02388..4ce624d90060 100644 --- a/cvat-ui/src/components/import-dataset-modal/styles.scss +++ b/cvat-ui/src/components/import-dataset-modal/styles.scss @@ -17,7 +17,7 @@ color: $text-color-secondary; } -.cvat-modal-import-project-status .ant-modal-body { +.cvat-modal-import-dataset-status .ant-modal-body { display: flex; align-items: center; flex-flow: column; diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index 9a663fa2843a..b0212bf7e1e1 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -3,15 +3,14 @@ // SPDX-License-Identifier: MIT import { ImportActions, ImportActionTypes } from 'actions/import-actions'; -import deepCopy from 'utils/deep-copy'; import { ImportState } from './interfaces'; const defaultState: ImportState = { - projects: {}, progress: 0.0, status: '', instance: null, + format: null, modalVisible: false, }; @@ -31,14 +30,11 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor }; } case ImportActionTypes.IMPORT_DATASET: { - const { instance, format } = action.payload; - const activities = deepCopy(state.projects); - - activities[instance.id] = format; + const { format } = action.payload; return { ...state, - projects: activities, + format, }; } case ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS: { @@ -51,14 +47,9 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor } case ImportActionTypes.IMPORT_DATASET_FAILED: case ImportActionTypes.IMPORT_DATASET_SUCCESS: { - const { instance } = action.payload; - const activities = deepCopy(state.projects); - - delete activities[instance.id]; - return { ...state, - projects: activities, + format: null, }; } default: diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 1e00f58a52d0..79900d28aa38 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -116,9 +116,7 @@ export interface ExportState { } export interface ImportState { - projects: { - [pid: number]: string; - }; + format: string | null; progress: number; status: string; instance: any; From b98124ecf556bf5be2867dd22bc55200faaadf4d Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 16 Nov 2021 20:35:06 +0300 Subject: [PATCH 43/57] Fixed callback arguments --- cvat-core/src/server-proxy.js | 2 +- cvat-ui/src/actions/import-actions.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index e3cc517cb5d9..382d54fdd68b 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -513,7 +513,7 @@ proxy: config.proxy, }); if (response.status === 202) { - if (callback) callback(response.data); + if (callback) callback(response.data.progress, response.data.message); setTimeout(requestStatus, 3000); } else if (response.status === 201) { resolve(); diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 46fa40ec9242..1fae27082921 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -43,8 +43,8 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T throw Error('Only one importing of dataset allowed at the same time'); } dispatch(importActions.importDataset(format)); - await instance.annotations.importDataset(format, file, (response: any) => ( - dispatch(importActions.importDatasetUpdateStatus(response.progress * 100, response.message)) + await instance.annotations.importDataset(format, file, (progress: number, message: string) => ( + dispatch(importActions.importDatasetUpdateStatus(progress * 100, message)) )); } catch (error) { dispatch(importActions.importDatasetFailed(instance, error)); From 18068b4579b616f62d1aea03fa1e80bdd092cbe1 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 19 Nov 2021 23:28:56 +0300 Subject: [PATCH 44/57] Added other simple formats --- cvat/apps/dataset_manager/bindings.py | 2 +- cvat/apps/dataset_manager/formats/cityscapes.py | 4 +++- cvat/apps/dataset_manager/formats/lfw.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index c2f4863e2b55..527bf2df8d1c 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -28,7 +28,7 @@ from .annotation import AnnotationIR, AnnotationManager, TrackManager -CVAT_INTERNAL_ATTRIBUTES = {'occluded', 'outside', 'keyframe', 'track_id'} +CVAT_INTERNAL_ATTRIBUTES = {'occluded', 'outside', 'keyframe', 'track_id', 'rotation'} class InstanceLabelData: Attribute = NamedTuple('Attribute', [('name', str), ('value', Any)]) diff --git a/cvat/apps/dataset_manager/formats/cityscapes.py b/cvat/apps/dataset_manager/formats/cityscapes.py index 607f849718cf..a6ec0833f534 100644 --- a/cvat/apps/dataset_manager/formats/cityscapes.py +++ b/cvat/apps/dataset_manager/formats/cityscapes.py @@ -32,7 +32,7 @@ def _export(dst_file, instance_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='Cityscapes', ext='ZIP', version='1.0') -def _import(src_file, instance_data): +def _import(src_file, instance_data, load_data_callback=None): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) @@ -44,4 +44,6 @@ def _import(src_file, instance_data): dataset = Dataset.import_from(tmp_dir, 'cityscapes', env=dm_env) dataset.transform('masks_to_polygons') + if load_data_callback is not None: + load_data_callback(dataset, instance_data) import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/lfw.py b/cvat/apps/dataset_manager/formats/lfw.py index 97cc9dcb18c3..6ec4cabafcee 100644 --- a/cvat/apps/dataset_manager/formats/lfw.py +++ b/cvat/apps/dataset_manager/formats/lfw.py @@ -14,12 +14,13 @@ @importer(name='LFW', ext='ZIP', version='1.0') -def _import(src_file, instance_data): +def _import(src_file, instance_data, load_data_callback=None): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'lfw') - + if load_data_callback is not None: + load_data_callback(dataset, instance_data) import_dm_annotations(dataset, instance_data) @exporter(name='LFW', ext='ZIP', version='1.0') From be6fbbc7f715e0f665b4780f737f6cb1aac195cc Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 23 Nov 2021 22:24:33 +0300 Subject: [PATCH 45/57] Fixed comments --- cvat-ui/src/actions/import-actions.ts | 8 ++++---- .../import-dataset-modal/import-dataset-modal.tsx | 8 ++++---- .../import-dataset-status-modal.tsx | 9 ++++----- .../src/components/import-dataset-modal/styles.scss | 2 +- cvat-ui/src/components/projects-page/actions-menu.tsx | 10 +++++----- cvat-ui/src/reducers/import-reducer.ts | 9 +++++---- cvat-ui/src/reducers/interfaces.ts | 2 +- cvat/apps/engine/views.py | 9 +++++---- cvat/requirements/base.txt | 1 + 9 files changed, 30 insertions(+), 28 deletions(-) diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 1fae27082921..99df4f758081 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -18,8 +18,8 @@ export enum ImportActionTypes { export const importActions = { openImportModal: (instance: any) => createAction(ImportActionTypes.OPEN_IMPORT_MODAL, { instance }), closeImportModal: () => createAction(ImportActionTypes.CLOSE_IMPORT_MODAL), - importDataset: (format: string) => ( - createAction(ImportActionTypes.IMPORT_DATASET, { format }) + importDataset: (projectId: number) => ( + createAction(ImportActionTypes.IMPORT_DATASET, { id: projectId }) ), importDatasetSuccess: () => ( createAction(ImportActionTypes.IMPORT_DATASET_SUCCESS) @@ -39,10 +39,10 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T async (dispatch, getState) => { try { const state: CombinedState = getState(); - if (state.import.format !== null) { + if (state.import.importingId !== null) { throw Error('Only one importing of dataset allowed at the same time'); } - dispatch(importActions.importDataset(format)); + dispatch(importActions.importDataset(instance.id)); await instance.annotations.importDataset(format, file, (progress: number, message: string) => ( dispatch(importActions.importDatasetUpdateStatus(progress * 100, message)) )); diff --git a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx index 3367f8520b34..48ee2e42ab3b 100644 --- a/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx +++ b/cvat-ui/src/components/import-dataset-modal/import-dataset-modal.tsx @@ -32,15 +32,15 @@ function ImportDatasetModal(): JSX.Element { const [file, setFile] = useState(null); const modalVisible = useSelector((state: CombinedState) => state.import.modalVisible); const instance = useSelector((state: CombinedState) => state.import.instance); - const format = useSelector((state: CombinedState) => state.import.format); + const currentImportId = useSelector((state: CombinedState) => state.import.importingId); const importers = useSelector((state: CombinedState) => state.formats.annotationFormats.loaders); const dispatch = useDispatch(); - const closeModal = (): void => { + const closeModal = useCallback((): void => { form.resetFields(); setFile(null); dispatch(importActions.closeImportModal()); - }; + }, [form]); const handleImport = useCallback( (values: FormValues): void => { @@ -105,7 +105,7 @@ function ImportDatasetModal(): JSX.Element { ) .map( (importer: any): JSX.Element => { - const pending = format !== null; + const pending = currentImportId !== null; const disabled = !importer.enabled || pending; return ( state.import.instance); - const format = useSelector((state: CombinedState) => state.import.format); + const currentImportId = useSelector((state: CombinedState) => state.import.importingId); const progress = useSelector((state: CombinedState) => state.import.progress); const status = useSelector((state: CombinedState) => state.import.status); return ( .ant-select-item-option-content, +.cvat-modal-import-dataset-option-item > .ant-select-item-option-content, .cvat-modal-import-select .ant-select-selection-item { > span[role='img'] { color: $info-icon-color; diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index cda7771a314d..f7f16611225f 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; +import React, { useCallback } from 'react'; import { useDispatch } from 'react-redux'; import Modal from 'antd/lib/modal'; import Menu from 'antd/lib/menu'; @@ -20,7 +20,7 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { const dispatch = useDispatch(); - const onDeleteProject = (): void => { + const onDeleteProject = useCallback((): void => { Modal.confirm({ title: `The project ${projectInstance.id} will be deleted`, content: 'All related data (images, annotations) will be lost. Continue?', @@ -34,17 +34,17 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { }, okText: 'Delete', }); - }; + }, []); return ( - + dispatch(exportActions.openExportModal(projectInstance))}> Export dataset dispatch(importActions.openImportModal(projectInstance))}> Import dataset -
+ Delete diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index b0212bf7e1e1..70d9df267533 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -10,7 +10,7 @@ const defaultState: ImportState = { progress: 0.0, status: '', instance: null, - format: null, + importingId: null, modalVisible: false, }; @@ -30,11 +30,12 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor }; } case ImportActionTypes.IMPORT_DATASET: { - const { format } = action.payload; + const { id } = action.payload; return { ...state, - format, + importingId: id, + status: 'The file is being uploaded to the server', }; } case ImportActionTypes.IMPORT_DATASET_UPDATE_STATUS: { @@ -49,7 +50,7 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor case ImportActionTypes.IMPORT_DATASET_SUCCESS: { return { ...state, - format: null, + importingId: null, }; } default: diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 79900d28aa38..b5f588951c6d 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -116,7 +116,7 @@ export interface ExportState { } export interface ImportState { - format: string | null; + importingId: number | null; progress: number; status: string; instance: any; diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 39a79e291475..3ec534abbac6 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1731,11 +1731,12 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name): rq_job = queue.enqueue_call( func=rq_func, args=(pk, filename, format_name), - job_id=rq_id + job_id=rq_id, + meta={ + 'tmp_file': filename, + 'tmp_file_descriptor': fd, + }, ) - rq_job.meta['tmp_file'] = filename - rq_job.meta['tmp_file_descriptor'] = fd - rq_job.save_meta() else: return Response(status=status.HTTP_409_CONFLICT, data='Import job already exists') diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 06fbb6db7993..2d386ec6733d 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,3 +1,4 @@ +attrs==21.2.0 click==7.1.2 Django==3.1.13 django-appconf==1.0.4 From c82bffa129ef49fd30f627bb2de125e858753a8c Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 24 Nov 2021 00:45:25 +0300 Subject: [PATCH 46/57] Added openimages format import --- cvat/apps/dataset_manager/formats/openimages.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/openimages.py b/cvat/apps/dataset_manager/formats/openimages.py index aa2ceaff3fb5..1842693f069a 100644 --- a/cvat/apps/dataset_manager/formats/openimages.py +++ b/cvat/apps/dataset_manager/formats/openimages.py @@ -51,7 +51,7 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='Open Images V6', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data, load_data_callback=None): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) @@ -64,14 +64,14 @@ def _import(src_file, task_data): item_ids = list(find_item_ids(tmp_dir)) root_hint = find_dataset_root( - [DatasetItem(id=item_id) for item_id in item_ids], task_data) + [DatasetItem(id=item_id) for item_id in item_ids], instance_data) for item_id in item_ids: frame_info = None try: frame_id = match_dm_item(DatasetItem(id=item_id), - task_data, root_hint) - frame_info = task_data.frame_info[frame_id] + instance_data, root_hint) + frame_info = instance_data.frame_info[frame_id] except Exception: # nosec pass if frame_info is not None: @@ -80,6 +80,8 @@ def _import(src_file, task_data): dataset = Dataset.import_from(tmp_dir, 'open_images', image_meta=image_meta, env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + if load_data_callback is not None: + load_data_callback(dataset, instance_data) + import_dm_annotations(dataset, instance_data) From 91d7153985a3fd5e2252140ee62a9551c980020f Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 30 Nov 2021 13:36:44 +0300 Subject: [PATCH 47/57] Added simple tests --- cvat/apps/dataset_manager/formats/cvat.py | 5 +- cvat/apps/engine/tests/test_rest_api.py | 159 ++++++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 6bf9f6c8cf1d..8f42152ddcbe 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -76,8 +76,9 @@ def _get_subsets_from_anno(path): for ev, el in context: if ev == 'start': if el.tag == 'subsets': - subsets = el.text.split('\n') - return subsets + if el.text is not None: + subsets = el.text.split('\n') + return subsets if ev == 'end': if el.tag == 'meta': return [DEFAULT_SUBSET_NAME] diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index d9786cb742ed..5ae656b7faed 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1396,6 +1396,165 @@ def test_api_v1_projects_id_tasks_no_auth(self): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) +class ProjectImportExportAPITestCase(APITestCase): + def setUp(self) -> None: + self.client = APIClient() + self.tasks = [] + self.projects = [] + + @classmethod + def setUpTestData(cls) -> None: + create_db_users(cls) + + cls.media_data = [ + { + **{ + **{"client_files[{}]".format(i): generate_image_file("test_{}.jpg".format(i))[1] for i in range(10)}, + }, + **{ + "image_quality": 75, + }, + }, + { + **{ + **{"client_files[{}]".format(i): generate_image_file("test_{}.jpg".format(i))[1] for i in range(10)}, + }, + "image_quality": 75, + }, + ] + + def _create_tasks(self): + self.tasks = [] + + def _create_task(task_data, media_data): + response = self.client.post('/api/v1/tasks', data=task_data, format="json") + assert response.status_code == status.HTTP_201_CREATED + tid = response.data["id"] + + for media in media_data.values(): + if isinstance(media, io.BytesIO): + media.seek(0) + response = self.client.post("/api/v1/tasks/{}/data".format(tid), data=media_data) + assert response.status_code == status.HTTP_202_ACCEPTED + response = self.client.get("/api/v1/tasks/{}".format(tid)) + data_id = response.data["data"] + self.tasks.append({ + "id": tid, + "data_id": data_id, + }) + + task_data = [ + { + "name": "my task #1", + "owner_id": self.owner.id, + "assignee_id": self.assignee.id, + "overlap": 0, + "segment_size": 100, + "project_id": self.projects[0]["id"], + }, + { + "name": "my task #2", + "owner_id": self.owner.id, + "assignee_id": self.assignee.id, + "overlap": 1, + "segment_size": 3, + "project_id": self.projects[0]["id"], + }, + ] + + with ForceLogin(self.owner, self.client): + for data, media in zip(task_data, self.media_data): + _create_task(data, media) + + def _create_projects(self): + self.projects = [] + + def _create_project(project_data): + response = self.client.post('/api/v1/projects', data=project_data, format="json") + assert response.status_code == status.HTTP_201_CREATED + self.projects.append(response.data) + + project_data = [ + { + "name": "Project for export", + "owner_id": self.owner.id, + "assignee_id": self.assignee.id, + "labels": [ + { + "name": "car", + "color": "#ff00ff", + "attributes": [{ + "name": "bool_attribute", + "mutable": True, + "input_type": AttributeType.CHECKBOX, + "default_value": "true" + }], + }, { + "name": "person", + }, + ] + }, { + "name": "Project for import", + "owner_id": self.owner.id, + "assignee_id": self.assignee.id, + }, + ] + + with ForceLogin(self.owner, self.client): + for data in project_data: + _create_project(data) + + def _run_api_v1_projects_id_dataset_export(self, pid, user, query_params=""): + with ForceLogin(user, self.client): + response = self.client.get("/api/v1/projects/{}/dataset?{}".format(pid, query_params), format="json") + return response + + def _run_api_v1_projects_id_dataset_import(self, pid, user, data, f): + with ForceLogin(user, self.client): + response = self.client.post("/api/v1/projects/{}/dataset?format={}".format(pid, f), data=data, format="multipart") + return response + + def _run_api_v1_projects_id_dataset_import_status(self, pid, user): + with ForceLogin(user, self.client): + response = self.client.get("/api/v1/projects/{}/dataset?action=import_status".format(pid), format="json") + return response + + def test_api_v1_projects_id_export_import(self): + + self._create_projects() + self._create_tasks() + pid_export, pid_import = self.projects[0]["id"], self.projects[1]["id"] + response = self._run_api_v1_projects_id_dataset_export(pid_export, self.owner, "format=CVAT for images 1.1") + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + response = self._run_api_v1_projects_id_dataset_export(pid_export, self.owner, "format=CVAT for images 1.1") + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + response = self._run_api_v1_projects_id_dataset_export(pid_export, self.owner, "format=CVAT for images 1.1&action=download") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertTrue(response.streaming) + tmp_file = tempfile.NamedTemporaryFile(suffix=".zip") + tmp_file.write(b"".join(response.streaming_content)) + tmp_file.seek(0) + + import_data = { + "dataset_file": tmp_file, + } + + response = self._run_api_v1_projects_id_dataset_import(pid_import, self.owner, import_data, "CVAT 1.1") + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + response = self._run_api_v1_projects_id_dataset_import_status(pid_import, self.owner) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + def tearDown(self) -> None: + for task in self.tasks: + shutil.rmtree(os.path.join(settings.TASKS_ROOT, str(task["id"]))) + shutil.rmtree(os.path.join(settings.MEDIA_DATA_ROOT, str(task["data_id"]))) + for project in self.projects: + shutil.rmtree(os.path.join(settings.PROJECTS_ROOT, str(project["id"]))) + class TaskListAPITestCase(APITestCase): def setUp(self): self.client = APIClient() From 22c17d51c0d81d9187f90ae53539f1bf4bf81e86 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 1 Dec 2021 16:39:00 +0300 Subject: [PATCH 48/57] Reworked import progress --- cvat-core/src/server-proxy.js | 12 +++++++----- cvat-ui/src/actions/import-actions.ts | 2 +- cvat-ui/src/actions/tasks-actions.ts | 4 ++-- cvat/apps/dataset_manager/bindings.py | 2 +- cvat/apps/engine/serializers.py | 1 + cvat/apps/engine/task.py | 11 +++++------ cvat/apps/engine/views.py | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 4e36b95315d3..c11055053044 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -513,7 +513,7 @@ }; } - async function importDataset(id, format, file, callback) { + async function importDataset(id, format, file, onUpdate) { const { backendAPI } = config; const url = `${backendAPI}/projects/${id}/dataset`; @@ -527,7 +527,9 @@ proxy: config.proxy, }); if (response.status === 202) { - if (callback) callback(response.data.progress, response.data.message); + if (onUpdate && response.data.message !== '') { + onUpdate(response.data.message, response.data.progress || 0); + } setTimeout(requestStatus, 3000); } else if (response.status === 201) { resolve(); @@ -612,7 +614,7 @@ const response = await Axios.get(`${backendAPI}/tasks/${id}/status`); if (['Queued', 'Started'].includes(response.data.state)) { if (response.data.message !== '') { - onUpdate(response.data.message); + onUpdate(response.data.message, response.data.progress || 0); } setTimeout(checkStatus, 1000); } else if (response.data.state === 'Finished') { @@ -656,7 +658,7 @@ let response = null; - onUpdate('The task is being created on the server..'); + onUpdate('The task is being created on the server..', null); try { response = await Axios.post(`${backendAPI}/tasks`, JSON.stringify(taskSpec), { proxy: config.proxy, @@ -668,7 +670,7 @@ throw generateError(errorData); } - onUpdate('The data are being uploaded to the server..'); + onUpdate('The data are being uploaded to the server..', null); try { await Axios.post(`${backendAPI}/tasks/${response.data.id}/data`, taskData, { proxy: config.proxy, diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 99df4f758081..71c9a8eb7fcc 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -43,7 +43,7 @@ export const importDatasetAsync = (instance: any, format: string, file: File): T throw Error('Only one importing of dataset allowed at the same time'); } dispatch(importActions.importDataset(instance.id)); - await instance.annotations.importDataset(format, file, (progress: number, message: string) => ( + await instance.annotations.importDataset(format, file, (message: string, progress: number) => ( dispatch(importActions.importDatasetUpdateStatus(progress * 100, message)) )); } catch (error) { diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index c584c141c169..11fcc0c91344 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -413,8 +413,8 @@ export function createTaskAsync(data: any): ThunkAction, {}, {}, A dispatch(createTask()); try { - const savedTask = await taskInstance.save((status: string): void => { - dispatch(createTaskUpdateStatus(status)); + const savedTask = await taskInstance.save((status: string, progress: number): void => { + dispatch(createTaskUpdateStatus(status + (progress !== null ? ` ${Math.floor(progress * 100)}%` : ''))); }); dispatch(createTaskSuccess(savedTask.id)); } catch (error) { diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 3953cd912448..b9b50b075d79 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1529,7 +1529,7 @@ def load_dataset_data(project_annotation, dataset: Dataset, project_data): for subset_id, subset in enumerate(dataset.subsets().values()): job = rq.get_current_job() job.meta['status'] = 'Task from dataset is being created...' - job.meta['progress'] = subset_id / len(dataset.subsets().keys()) + job.meta['progress'] = (subset_id + job.meta.get('task_progress', 0.)) / len(dataset.subsets().keys()) job.save_meta() task_fields = { diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index b3d894002ca6..01df3525c89c 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -220,6 +220,7 @@ class RqStatusSerializer(serializers.Serializer): state = serializers.ChoiceField(choices=[ "Queued", "Started", "Finished", "Failed"]) message = serializers.CharField(allow_blank=True, default="") + progress = serializers.FloatField(max_value=100, default=0) class WriteOnceMixin: diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index eca82f22754e..ba099a7d25a8 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -354,12 +354,11 @@ def update_progress(progress): if not hasattr(update_progress, 'call_counter'): update_progress.call_counter = 0 - status_template = 'Images are being compressed {}' - if progress: - current_progress = '{}%'.format(round(progress * 100)) - else: - current_progress = '{}'.format(progress_animation[update_progress.call_counter]) - job.meta['status'] = status_template.format(current_progress) + status_message = 'Images are being compressed' + if not progress: + status_message = '{} {}'.format(status_message, progress_animation[update_progress.call_counter]) + job.meta['status'] = status_message + job.meta['task_progress'] = progress or 0. job.save_meta() update_progress.call_counter = (update_progress.call_counter + 1) % len(progress_animation) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a03df23c4206..413bd02ba889 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -434,7 +434,6 @@ def _get_rq_response(queue, job_id): response['message'] = job.meta.get('status', '') response['progress'] = job.meta.get('progress', 0.) - return response class TaskFilter(filters.FilterSet): @@ -905,6 +904,7 @@ def _get_rq_response(queue, job_id): response = { "state": "Started" } if 'status' in job.meta: response['message'] = job.meta['status'] + response['progress'] = job.meta.get('task_progress', 0.) return response From e1f0861c2e24b57faef699c900726c4fce416590 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 8 Dec 2021 19:32:30 +0300 Subject: [PATCH 49/57] Fixed tracks importing in cvat format --- cvat-ui/src/reducers/import-reducer.ts | 2 ++ cvat/apps/dataset_manager/bindings.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cvat-ui/src/reducers/import-reducer.ts b/cvat-ui/src/reducers/import-reducer.ts index 70d9df267533..db851e83788c 100644 --- a/cvat-ui/src/reducers/import-reducer.ts +++ b/cvat-ui/src/reducers/import-reducer.ts @@ -50,6 +50,8 @@ export default (state: ImportState = defaultState, action: ImportActions): Impor case ImportActionTypes.IMPORT_DATASET_SUCCESS: { return { ...state, + progress: defaultState.progress, + status: defaultState.status, importingId: null, }; } diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index b9b50b075d79..e6e8ee9b02cc 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1398,6 +1398,8 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr if isinstance(instance_data, ProjectData): for sub_dataset, task_data in instance_data.split_dataset(dm_dataset): + # FIXME: temporary workaround for cvat format, will be removed after migration importer to datumaro + sub_dataset._format = dm_dataset.format import_dm_annotations(sub_dataset, task_data) return From e24301ef3ebfcfc11e3eed1c2dddf05e33597245 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 9 Dec 2021 06:04:59 +0300 Subject: [PATCH 50/57] Fixed dataset root --- cvat/apps/dataset_manager/bindings.py | 4 ++++ cvat/apps/dataset_manager/formats/cvat.py | 14 ++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index e6e8ee9b02cc..8161098781b0 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1557,4 +1557,8 @@ def load_dataset_data(project_annotation, dataset: Dataset, project_data): dataset_files['media'] += \ list(map(lambda ri: ri.path, dataset_item.related_images)) + shortes_path = min(dataset_files['media'], key=lambda x: len(Path(x).parts), default=None) + if shortes_path is not None: + dataset_files['data_root'] = str(Path(shortes_path).parent.absolute()) + osp.sep + project_annotation.add_task(task_fields, dataset_files, project_data) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 8f42152ddcbe..8f1db9148022 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -89,21 +89,19 @@ def _get_subsets_from_anno(path): def _parse_images(image_dir, subsets): items = OrderedDict() - if subsets == [DEFAULT_SUBSET_NAME] and not osp.isdir(osp.join(image_dir, DEFAULT_SUBSET_NAME)): - for file in sorted(glob(osp.join(image_dir, '*.*')), key=osp.basename): + def parse_image_dir(image_dir): + for file in sorted(glob(image_dir), key=osp.path.basename): name, ext = osp.splitext(osp.basename(file)) if ext.lower() in CvatPath.MEDIA_EXTS: items[(None, name)] = DatasetItem(id=name, annotations=[], image=Image(path=file), subset=DEFAULT_SUBSET_NAME, ) + + if subsets == [DEFAULT_SUBSET_NAME] and not osp.isdir(osp.join(image_dir, DEFAULT_SUBSET_NAME)): + parse_image_dir(osp.join(image_dir, '*.*')) else: for subset in subsets: - for file in sorted(glob(osp.join(image_dir, subset, '*.*')), key=osp.basename): - name, ext = osp.splitext(osp.basename(file)) - if ext.lower() in CvatPath.MEDIA_EXTS: - items[(subset, name)] = DatasetItem(id=name, annotations=[], - image=Image(path=file), subset=subset, - ) + parse_image_dir(osp.path.join(image_dir, subset, '*.*')) return items @classmethod From edda65e3ee9a3b3e82bb3ae9f9f47756e4485c29 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 9 Dec 2021 10:53:52 +0300 Subject: [PATCH 51/57] Fixed methods --- cvat/apps/dataset_manager/formats/cvat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 8f1db9148022..d79cdd1d6102 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -90,7 +90,7 @@ def _parse_images(image_dir, subsets): items = OrderedDict() def parse_image_dir(image_dir): - for file in sorted(glob(image_dir), key=osp.path.basename): + for file in sorted(glob(image_dir), key=osp.basename): name, ext = osp.splitext(osp.basename(file)) if ext.lower() in CvatPath.MEDIA_EXTS: items[(None, name)] = DatasetItem(id=name, annotations=[], @@ -101,7 +101,7 @@ def parse_image_dir(image_dir): parse_image_dir(osp.join(image_dir, '*.*')) else: for subset in subsets: - parse_image_dir(osp.path.join(image_dir, subset, '*.*')) + parse_image_dir(osp.join(image_dir, subset, '*.*')) return items @classmethod From 4b4dff7c36423a5c10e1d6a8ecf0a39e96c61c91 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 13 Dec 2021 11:57:22 +0300 Subject: [PATCH 52/57] Fixed export status update --- cvat-ui/src/components/export-dataset/export-dataset-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx index 4eda40a4732d..32e5654c9109 100644 --- a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -55,7 +55,7 @@ function ExportDatasetModal(): JSX.Element { useEffect(() => { initActivities(); - }, [instance?.id, instance instanceof core.classes.Project]); + }, [instance?.id, instance instanceof core.classes.Project, taskExportActivities, projectExportActivities]); const closeModal = (): void => { form.resetFields(); From c3f310c11cea0132562bd692a9e4cbd1df66e192 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 13 Dec 2021 13:11:50 +0300 Subject: [PATCH 53/57] Fixed name --- cvat/apps/engine/task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index e90813f44952..e2d6c8fe483c 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -307,7 +307,7 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): db_data.start_frame = 0 data['stop_frame'] = None db_data.frame_filter = '' - if isDatasetImport and media_type != 'video' and db_data.storage_method == models.StorageMethodChoice.CACHE: + if isBackupRestore and media_type != 'video' and db_data.storage_method == models.StorageMethodChoice.CACHE: # we should sort media_files according to the manifest content sequence manifest = ImageManifestManager(db_data.get_manifest_path()) manifest.set_index() @@ -324,9 +324,9 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): del sorted_media_files data['sorting_method'] = models.SortingMethod.PREDEFINED source_paths=[os.path.join(upload_dir, f) for f in media_files] - if manifest_file and not isDatasetImport and data['sorting_method'] in {models.SortingMethod.RANDOM, models.SortingMethod.PREDEFINED}: + if manifest_file and not isBackupRestore and data['sorting_method'] in {models.SortingMethod.RANDOM, models.SortingMethod.PREDEFINED}: raise Exception("It isn't supported to upload manifest file and use random sorting") - if isDatasetImport and db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM and \ + if isBackupRestore and db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM and \ data['sorting_method'] in {models.SortingMethod.RANDOM, models.SortingMethod.PREDEFINED}: raise Exception("It isn't supported to import the task that was created without cache but with random/predefined sorting") From 0fe7619ce85deec0fb2a513cc3d7c433538cde08 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 14 Dec 2021 10:26:49 +0300 Subject: [PATCH 54/57] Fixed import error with subsets --- cvat/apps/dataset_manager/bindings.py | 2 +- cvat/apps/dataset_manager/formats/cvat.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 8161098781b0..fe2d63759e74 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -752,7 +752,7 @@ def _init_meta(self): ])) for db_label in self._label_mapping.values() ]), - ("subsets", '\n'.join(self._subsets)), + ("subsets", '\n'.join([s if s else datum_extractor.DEFAULT_SUBSET_NAME for s in self._subsets])), ("owner", OrderedDict([ ("username", self._db_project.owner.username), diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index d79cdd1d6102..1bc472f7a5c2 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -89,19 +89,19 @@ def _get_subsets_from_anno(path): def _parse_images(image_dir, subsets): items = OrderedDict() - def parse_image_dir(image_dir): + def parse_image_dir(image_dir, subset): for file in sorted(glob(image_dir), key=osp.basename): name, ext = osp.splitext(osp.basename(file)) if ext.lower() in CvatPath.MEDIA_EXTS: - items[(None, name)] = DatasetItem(id=name, annotations=[], - image=Image(path=file), subset=DEFAULT_SUBSET_NAME, + items[(subset, name)] = DatasetItem(id=name, annotations=[], + image=Image(path=file), subset=subset or DEFAULT_SUBSET_NAME, ) if subsets == [DEFAULT_SUBSET_NAME] and not osp.isdir(osp.join(image_dir, DEFAULT_SUBSET_NAME)): - parse_image_dir(osp.join(image_dir, '*.*')) + parse_image_dir(osp.join(image_dir, '*.*'), None) else: for subset in subsets: - parse_image_dir(osp.join(image_dir, subset, '*.*')) + parse_image_dir(osp.join(image_dir, subset, '*.*'), subset) return items @classmethod From 48f044b4c55e649a3d4d39ccf1c6b3674620789a Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 14 Dec 2021 14:08:57 +0300 Subject: [PATCH 55/57] Fixed label colors import --- cvat/apps/dataset_manager/bindings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index fe2d63759e74..087abe6d8d9c 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1511,14 +1511,14 @@ def import_dm_annotations(dm_dataset: Dataset, instance_data: Union[TaskData, Pr def import_labels_to_project(project_annotation, dataset: Dataset): labels = [] - label_names = [] + label_colors = [] for label in dataset.categories()[datum_annotation.AnnotationType.label].items: db_label = Label( name=label.name, - color=get_label_color(label.name, label_names) + color=get_label_color(label.name, label_colors) ) labels.append(db_label) - label_names.append(label.name) + label_colors.append(label.color) project_annotation.add_labels(labels) def load_dataset_data(project_annotation, dataset: Dataset, project_data): From 285d2d04091661ec293f7e4fa953635d23622182 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Tue, 14 Dec 2021 14:52:03 +0300 Subject: [PATCH 56/57] Fixed label colors --- cvat/apps/dataset_manager/bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 087abe6d8d9c..64dc1bb55f85 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1518,7 +1518,7 @@ def import_labels_to_project(project_annotation, dataset: Dataset): color=get_label_color(label.name, label_colors) ) labels.append(db_label) - label_colors.append(label.color) + label_colors.append(db_label.color) project_annotation.add_labels(labels) def load_dataset_data(project_annotation, dataset: Dataset, project_data): From c1110daa17cdcee5f44a812b15682f24a197a95c Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 17 Dec 2021 12:06:59 +0300 Subject: [PATCH 57/57] Added cvat-ui version --- cvat-ui/package-lock.json | 4 ++-- cvat-ui/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index ebe48e3d0a2b..34f85b08742b 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "cvat-ui", - "version": "1.29.0", + "version": "1.30.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cvat-ui", - "version": "1.29.0", + "version": "1.30.0", "license": "MIT", "dependencies": { "@ant-design/icons": "^4.6.3", diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 5030d6b13afc..00e91ff2ca18 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.29.0", + "version": "1.30.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": {