From f18b1cb82dba366502e481a00900390337051372 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 6 Aug 2021 16:26:37 +0300 Subject: [PATCH] Project: export as a dataset (#3365) --- .github/workflows/main.yml | 3 +- CHANGELOG.md | 1 + cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-core/src/annotations.js | 43 +- cvat-core/src/api.js | 3 +- cvat-core/src/project-implementation.js | 74 ++ cvat-core/src/project.js | 86 +- cvat-core/src/server-proxy.js | 55 +- cvat-core/src/session.js | 50 +- cvat-ui/src/actions/export-actions.ts | 49 + cvat-ui/src/actions/tasks-actions.ts | 114 --- .../components/actions-menu/actions-menu.tsx | 25 +- .../components/actions-menu/dump-submenu.tsx | 54 -- .../actions-menu/export-submenu.tsx | 47 - .../objects-side-bar/objects-side-bar.tsx | 18 +- .../top-bar/annotation-menu.tsx | 26 +- .../export-dataset/export-dataset-modal.tsx | 145 +++ .../src/components/export-dataset/styles.scss | 13 + .../components/project-page/project-page.tsx | 2 + .../components/projects-page/actions-menu.tsx | 8 +- .../projects-page/projects-page.tsx | 6 +- .../src/components/task-page/task-page.tsx | 2 + .../src/components/tasks-page/tasks-page.tsx | 2 + .../containers/actions-menu/actions-menu.tsx | 44 +- .../top-bar/annotation-menu.tsx | 46 +- cvat-ui/src/reducers/export-reducer.ts | 67 ++ cvat-ui/src/reducers/formats-reducer.ts | 2 +- cvat-ui/src/reducers/interfaces.ts | 20 +- cvat-ui/src/reducers/notifications-reducer.ts | 30 +- cvat-ui/src/reducers/root-reducer.ts | 4 +- cvat-ui/src/reducers/tasks-reducer.ts | 80 -- cvat-ui/src/utils/deep-copy.ts | 21 + cvat/apps/dataset_manager/annotation.py | 3 + cvat/apps/dataset_manager/bindings.py | 870 +++++++++++++----- cvat/apps/dataset_manager/formats/camvid.py | 14 +- cvat/apps/dataset_manager/formats/coco.py | 14 +- cvat/apps/dataset_manager/formats/cvat.py | 144 ++- .../formats/datumaro/__init__.py | 39 +- cvat/apps/dataset_manager/formats/icdar.py | 32 +- cvat/apps/dataset_manager/formats/imagenet.py | 12 +- cvat/apps/dataset_manager/formats/labelme.py | 12 +- .../dataset_manager/formats/market1501.py | 12 +- cvat/apps/dataset_manager/formats/mask.py | 14 +- cvat/apps/dataset_manager/formats/mot.py | 8 +- cvat/apps/dataset_manager/formats/mots.py | 8 +- .../dataset_manager/formats/pascal_voc.py | 19 +- .../dataset_manager/formats/pointcloud.py | 5 +- cvat/apps/dataset_manager/formats/tfrecord.py | 12 +- cvat/apps/dataset_manager/formats/utils.py | 5 +- .../dataset_manager/formats/velodynepoint.py | 5 +- cvat/apps/dataset_manager/formats/vggface2.py | 12 +- .../apps/dataset_manager/formats/widerface.py | 12 +- cvat/apps/dataset_manager/formats/yolo.py | 8 +- cvat/apps/dataset_manager/project.py | 71 ++ .../tests/assets/projects.json | 55 ++ .../dataset_manager/tests/assets/tasks.json | 23 +- .../tests/test_rest_api_formats.py | 137 ++- cvat/apps/dataset_manager/views.py | 73 +- cvat/apps/engine/serializers.py | 8 +- cvat/apps/engine/task.py | 3 + cvat/apps/engine/views.py | 99 +- .../basics/creating_an_annotation_task.md | 92 +- .../case_94_move_task_between_projects.js | 38 +- .../case_95_move_task_to_project.js | 6 +- .../case_52_dump_upload_annotation.js | 7 +- ..._import_annotations_frames_dots_in_name.js | 13 +- .../case_97_export_import_task.js | 13 +- .../actions_tasks3/case_47_export_dataset.js | 17 +- .../actions_tasks3/case_90_context_image.js | 42 +- ...mp_upload_annotation_point_cloud_format.js | 15 +- ...pload_annotation_velodyne_points_format.js | 12 +- ...3_canvas3d_functionality_export_dataset.js | 33 +- .../issue_1568_cuboid_dump_annotation.js | 35 +- tests/cypress/support/commands.js | 11 +- 75 files changed, 2042 insertions(+), 1165 deletions(-) create mode 100644 cvat-core/src/project-implementation.js create mode 100644 cvat-ui/src/actions/export-actions.ts delete mode 100644 cvat-ui/src/components/actions-menu/dump-submenu.tsx delete mode 100644 cvat-ui/src/components/actions-menu/export-submenu.tsx create mode 100644 cvat-ui/src/components/export-dataset/export-dataset-modal.tsx create mode 100644 cvat-ui/src/components/export-dataset/styles.scss create mode 100644 cvat-ui/src/reducers/export-reducer.ts create mode 100644 cvat-ui/src/utils/deep-copy.ts create mode 100644 cvat/apps/dataset_manager/project.py create mode 100644 cvat/apps/dataset_manager/tests/assets/projects.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 877a095f59ec..9374a9758a94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,8 +5,7 @@ on: - 'master' - 'develop' pull_request: - branches: - - '*' + jobs: Unit_testing: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index cb0c298be44f..4e3f6fadc51c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Notification if the browser does not support nesassary API +- Added ability to export project as a dataset () - Additional inline tips in interactors with demo gifs () ### Changed diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 6daac70214eb..e0b9594df794 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.13.3", + "version": "3.14.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 1cf4c506da2b..9c169475461a 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.13.3", + "version": "3.14.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 991ee33eed75..07ce90f9e2f0 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -8,8 +8,9 @@ const AnnotationsSaver = require('./annotations-saver'); const AnnotationsHistory = require('./annotations-history'); const { checkObjectType } = require('./common'); - const { Task } = require('./session'); - const { Loader, Dumper } = require('./annotation-formats'); + const { Project } = require('./project'); + const { Task, Job } = require('./session'); + const { Loader } = require('./annotation-formats'); const { ScriptingError, DataError, ArgumentError } = require('./exceptions'); const jobCache = new WeakMap(); @@ -50,6 +51,7 @@ stopFrame, frameMeta, }); + // eslint-disable-next-line no-unsanitized/method collection.import(rawAnnotations); const saver = new AnnotationsSaver(rawAnnotations.version, collection, session); @@ -232,27 +234,12 @@ await serverProxy.annotations.uploadAnnotations(sessionType, session.id, file, loader.name); } - async function dumpAnnotations(session, name, dumper) { - if (!(dumper instanceof Dumper)) { - throw new ArgumentError('A dumper must be instance of Dumper class'); - } - - let result = null; - const sessionType = session instanceof Task ? 'task' : 'job'; - if (sessionType === 'job') { - result = await serverProxy.annotations.dumpAnnotations(session.task.id, name, dumper.name); - } else { - result = await serverProxy.annotations.dumpAnnotations(session.id, name, dumper.name); - } - - return result; - } - function importAnnotations(session, data) { const sessionType = session instanceof Task ? 'task' : 'job'; const cache = getCache(sessionType); if (cache.has(session)) { + // eslint-disable-next-line no-unsanitized/method return cache.get(session).collection.import(data); } @@ -274,16 +261,25 @@ ); } - async function exportDataset(session, format) { + async function exportDataset(instance, format, name, saveImages = false) { if (!(format instanceof String || typeof format === 'string')) { throw new ArgumentError('Format must be a string'); } - if (!(session instanceof Task)) { - throw new ArgumentError('A dataset can only be created from a task'); + if (!(instance instanceof Task || instance instanceof Project || instance instanceof Job)) { + throw new ArgumentError('A dataset can only be created from a job, task or project'); + } + if (typeof saveImages !== 'boolean') { + throw new ArgumentError('Save images parameter must be a boolean'); } let result = null; - result = await serverProxy.tasks.exportDataset(session.id, format); + if (instance instanceof Task) { + result = await serverProxy.tasks.exportDataset(instance.id, format, name, saveImages); + } else if (instance instanceof Job) { + result = await serverProxy.tasks.exportDataset(instance.task.id, format, name, saveImages); + } else { + result = await serverProxy.projects.exportDataset(instance.id, format, name, saveImages); + } return result; } @@ -367,7 +363,6 @@ annotationsStatistics, selectObject, uploadAnnotations, - dumpAnnotations, importAnnotations, exportAnnotations, exportDataset, diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index 7067d560ab93..d517b444caa3 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -18,6 +18,7 @@ function build() { const Review = require('./review'); const { Job, Task } = require('./session'); const { Project } = require('./project'); + const implementProject = require('./project-implementation'); const { Attribute, Label } = require('./labels'); const MLModel = require('./ml-model'); const { FrameData } = require('./frames'); @@ -754,7 +755,7 @@ function build() { */ classes: { User, - Project, + Project: implementProject(Project), Task, Job, Log, diff --git a/cvat-core/src/project-implementation.js b/cvat-core/src/project-implementation.js new file mode 100644 index 000000000000..c5bb2387099d --- /dev/null +++ b/cvat-core/src/project-implementation.js @@ -0,0 +1,74 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +(() => { + const serverProxy = require('./server-proxy'); + const { getPreview } = require('./frames'); + + const { Project } = require('./project'); + const { exportDataset } = require('./annotations'); + + function implementProject(projectClass) { + projectClass.prototype.save.implementation = async function () { + const trainingProjectCopy = this.trainingProject; + if (typeof this.id !== 'undefined') { + // project has been already created, need to update some data + const projectData = { + name: this.name, + assignee_id: this.assignee ? this.assignee.id : null, + bug_tracker: this.bugTracker, + labels: [...this._internalData.labels.map((el) => el.toJSON())], + }; + + if (trainingProjectCopy) { + projectData.training_project = trainingProjectCopy; + } + + await serverProxy.projects.save(this.id, projectData); + return this; + } + + // initial creating + const projectSpec = { + name: this.name, + labels: [...this.labels.map((el) => el.toJSON())], + }; + + if (this.bugTracker) { + projectSpec.bug_tracker = this.bugTracker; + } + + if (trainingProjectCopy) { + projectSpec.training_project = trainingProjectCopy; + } + + const project = await serverProxy.projects.create(projectSpec); + return new Project(project); + }; + + projectClass.prototype.delete.implementation = async function () { + const result = await serverProxy.projects.delete(this.id); + return result; + }; + + projectClass.prototype.preview.implementation = async function () { + if (!this._internalData.task_ids.length) { + return ''; + } + const frameData = await getPreview(this._internalData.task_ids[0]); + return frameData; + }; + + projectClass.prototype.annotations.exportDataset.implementation = async function ( + format, saveImages, customName, + ) { + const result = exportDataset(this, format, customName, saveImages); + return result; + }; + + return projectClass; + } + + module.exports = implementProject; +})(); diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js index b66eab496525..bd25fc0f1b13 100644 --- a/cvat-core/src/project.js +++ b/cvat-core/src/project.js @@ -4,11 +4,9 @@ (() => { const PluginRegistry = require('./plugins'); - const serverProxy = require('./server-proxy'); const { ArgumentError } = require('./exceptions'); const { Task } = require('./session'); const { Label } = require('./labels'); - const { getPreview } = require('./frames'); const User = require('./user'); /** @@ -203,7 +201,7 @@ }, }, /** - * Tasks linked with the project + * Tasks related with the project * @name tasks * @type {module:API.cvat.classes.Task[]} * @memberof module:API.cvat.classes.Project @@ -214,7 +212,7 @@ get: () => [...data.tasks], }, /** - * Subsets array for linked tasks + * Subsets array for related tasks * @name subsets * @type {string[]} * @memberof module:API.cvat.classes.Project @@ -254,6 +252,13 @@ }, }), ); + + // When we call a function, for example: project.annotations.get() + // In the method get we lose the project context + // So, we need return it + this.annotations = { + exportDataset: Object.getPrototypeOf(this).annotations.exportDataset.bind(this), + }; } /** @@ -289,7 +294,7 @@ } /** - * Method deletes a task from a server + * Method deletes a project from a server * @method delete * @memberof module:API.cvat.classes.Project * @readonly @@ -304,57 +309,28 @@ } } + Object.defineProperties( + Project.prototype, + Object.freeze({ + annotations: Object.freeze({ + value: { + async exportDataset(format, saveImages, customName = '') { + const result = await PluginRegistry.apiWrapper.call( + this, + Project.prototype.annotations.exportDataset, + format, + saveImages, + customName, + ); + return result; + }, + }, + writable: true, + }), + }), + ); + module.exports = { Project, }; - - Project.prototype.save.implementation = async function () { - const trainingProjectCopy = this.trainingProject; - if (typeof this.id !== 'undefined') { - // project has been already created, need to update some data - const projectData = { - name: this.name, - assignee_id: this.assignee ? this.assignee.id : null, - bug_tracker: this.bugTracker, - labels: [...this._internalData.labels.map((el) => el.toJSON())], - }; - - if (trainingProjectCopy) { - projectData.training_project = trainingProjectCopy; - } - - await serverProxy.projects.save(this.id, projectData); - return this; - } - - // initial creating - const projectSpec = { - name: this.name, - labels: [...this.labels.map((el) => el.toJSON())], - }; - - if (this.bugTracker) { - projectSpec.bug_tracker = this.bugTracker; - } - - if (trainingProjectCopy) { - projectSpec.training_project = trainingProjectCopy; - } - - const project = await serverProxy.projects.create(projectSpec); - return new Project(project); - }; - - Project.prototype.delete.implementation = async function () { - const result = await serverProxy.projects.delete(this.id); - return result; - }; - - Project.prototype.preview.implementation = async function () { - if (!this._internalData.task_ids.length) { - return ''; - } - const frameData = await getPreview(this._internalData.task_ids[0]); - return frameData; - }; })(); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 524ceeafd701..3d914ce9db7a 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -465,29 +465,39 @@ } } - async function exportDataset(id, format) { - const { backendAPI } = config; - let url = `${backendAPI}/tasks/${id}/dataset?format=${format}`; + function exportDataset(instanceType) { + return async function (id, format, name, saveImages) { + const { backendAPI } = config; + const baseURL = `${backendAPI}/${instanceType}/${id}/${saveImages ? 'dataset' : 'annotations'}`; + let query = `format=${encodeURIComponent(format)}`; + if (name) { + const filename = name.replace(/\//g, '_'); + query += `&filename=${encodeURIComponent(filename)}`; + } + let url = `${baseURL}?${query}`; - return new Promise((resolve, reject) => { - async function request() { - try { - const response = await Axios.get(`${url}`, { + return new Promise((resolve, reject) => { + async function request() { + Axios.get(`${url}`, { proxy: config.proxy, - }); - if (response.status === 202) { - setTimeout(request, 3000); - } else { - url = `${url}&action=download`; - resolve(url); - } - } catch (errorData) { - reject(generateError(errorData)); + }) + .then((response) => { + if (response.status === 202) { + setTimeout(request, 3000); + } else { + query = `${query}&action=download`; + url = `${baseURL}?${query}`; + resolve(url); + } + }) + .catch((errorData) => { + reject(generateError(errorData)); + }); } - } - setTimeout(request); - }); + setTimeout(request); + }); + }; } async function exportTask(id) { @@ -1135,7 +1145,9 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; + const predicate = () => ( + !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId + ); if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { @@ -1199,6 +1211,7 @@ save: saveProject, create: createProject, delete: deleteProject, + exportDataset: exportDataset('projects'), }), writable: false, }, @@ -1209,7 +1222,7 @@ saveTask, createTask, deleteTask, - exportDataset, + exportDataset: exportDataset('tasks'), exportTask, importTask, }), diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index a1136030fd9e..c5366ea1d5f5 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -42,16 +42,6 @@ return result; }, - async dump(dumper, name = null) { - const result = await PluginRegistry.apiWrapper.call( - this, - prototype.annotations.dump, - dumper, - name, - ); - return result; - }, - async statistics() { const result = await PluginRegistry.apiWrapper.call(this, prototype.annotations.statistics); return result; @@ -148,11 +138,13 @@ return result; }, - async exportDataset(format) { + async exportDataset(format, saveImages, customName = '') { const result = await PluginRegistry.apiWrapper.call( this, prototype.annotations.exportDataset, format, + saveImages, + customName, ); return result; }, @@ -329,21 +321,6 @@ * @instance * @async */ - /** - * Dump of annotations to a file. - * Method always dumps annotations for a whole task. - * @method dump - * @memberof Session.annotations - * @param {module:API.cvat.classes.Dumper} dumper - a dumper - * @param {string} [name = null] - a name of a file with annotations - * which will be used to dump - * @returns {string} URL which can be used in order to get a dump file - * @throws {module:API.cvat.exceptions.PluginError} - * @throws {module:API.cvat.exceptions.ServerError} - * @throws {module:API.cvat.exceptions.ArgumentError} - * @instance - * @async - */ /** * Collect short statistics about a task or a job. * @method statistics @@ -877,7 +854,6 @@ get: Object.getPrototypeOf(this).annotations.get.bind(this), put: Object.getPrototypeOf(this).annotations.put.bind(this), save: Object.getPrototypeOf(this).annotations.save.bind(this), - dump: Object.getPrototypeOf(this).annotations.dump.bind(this), merge: Object.getPrototypeOf(this).annotations.merge.bind(this), split: Object.getPrototypeOf(this).annotations.split.bind(this), group: Object.getPrototypeOf(this).annotations.group.bind(this), @@ -1575,7 +1551,6 @@ get: Object.getPrototypeOf(this).annotations.get.bind(this), put: Object.getPrototypeOf(this).annotations.put.bind(this), save: Object.getPrototypeOf(this).annotations.save.bind(this), - dump: Object.getPrototypeOf(this).annotations.dump.bind(this), merge: Object.getPrototypeOf(this).annotations.merge.bind(this), split: Object.getPrototypeOf(this).annotations.split.bind(this), group: Object.getPrototypeOf(this).annotations.group.bind(this), @@ -1715,7 +1690,6 @@ selectObject, annotationsStatistics, uploadAnnotations, - dumpAnnotations, importAnnotations, exportAnnotations, exportDataset, @@ -1948,13 +1922,8 @@ return result; }; - Job.prototype.annotations.dump.implementation = async function (dumper, name) { - const result = await dumpAnnotations(this, name, dumper); - return result; - }; - - Job.prototype.annotations.exportDataset.implementation = async function (format) { - const result = await exportDataset(this.task, format); + Job.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { + const result = await exportDataset(this.task, format, customName, saveImages); return result; }; @@ -2252,11 +2221,6 @@ return result; }; - Task.prototype.annotations.dump.implementation = async function (dumper, name) { - const result = await dumpAnnotations(this, name, dumper); - return result; - }; - Task.prototype.annotations.import.implementation = function (data) { const result = importAnnotations(this, data); return result; @@ -2267,8 +2231,8 @@ return result; }; - Task.prototype.annotations.exportDataset.implementation = async function (format) { - const result = await exportDataset(this, format); + Task.prototype.annotations.exportDataset.implementation = async function (format, saveImages, customName) { + const result = await exportDataset(this, format, customName, saveImages); return result; }; diff --git a/cvat-ui/src/actions/export-actions.ts b/cvat-ui/src/actions/export-actions.ts new file mode 100644 index 000000000000..d5a2b801b905 --- /dev/null +++ b/cvat-ui/src/actions/export-actions.ts @@ -0,0 +1,49 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ActionUnion, createAction, ThunkAction } from 'utils/redux'; + +export enum ExportActionTypes { + OPEN_EXPORT_MODAL = 'OPEN_EXPORT_MODAL', + CLOSE_EXPORT_MODAL = 'CLOSE_EXPORT_MODAL', + EXPORT_DATASET = 'EXPORT_DATASET', + EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', + EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', +} + +export const exportActions = { + openExportModal: (instance: any) => createAction(ExportActionTypes.OPEN_EXPORT_MODAL, { instance }), + closeExportModal: () => createAction(ExportActionTypes.CLOSE_EXPORT_MODAL), + exportDataset: (instance: any, format: string) => + createAction(ExportActionTypes.EXPORT_DATASET, { instance, format }), + exportDatasetSuccess: (instance: any, format: string) => + createAction(ExportActionTypes.EXPORT_DATASET_SUCCESS, { instance, format }), + exportDatasetFailed: (instance: any, format: string, error: any) => + createAction(ExportActionTypes.EXPORT_DATASET_FAILED, { + instance, + format, + error, + }), +}; + +export const exportDatasetAsync = ( + instance: any, + format: string, + name: string, + saveImages: boolean, +): ThunkAction => async (dispatch) => { + dispatch(exportActions.exportDataset(instance, format)); + + try { + const url = await instance.annotations.exportDataset(format, saveImages, name); + const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; + downloadAnchor.href = url; + downloadAnchor.click(); + dispatch(exportActions.exportDatasetSuccess(instance, format)); + } catch (error) { + dispatch(exportActions.exportDatasetFailed(instance, format, error)); + } +}; + +export type ExportActions = ActionUnion; diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index c0fabf97ecdf..468bfe8c3519 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -18,12 +18,6 @@ export enum TasksActionTypes { LOAD_ANNOTATIONS = 'LOAD_ANNOTATIONS', LOAD_ANNOTATIONS_SUCCESS = 'LOAD_ANNOTATIONS_SUCCESS', LOAD_ANNOTATIONS_FAILED = 'LOAD_ANNOTATIONS_FAILED', - DUMP_ANNOTATIONS = 'DUMP_ANNOTATIONS', - DUMP_ANNOTATIONS_SUCCESS = 'DUMP_ANNOTATIONS_SUCCESS', - DUMP_ANNOTATIONS_FAILED = 'DUMP_ANNOTATIONS_FAILED', - EXPORT_DATASET = 'EXPORT_DATASET', - EXPORT_DATASET_SUCCESS = 'EXPORT_DATASET_SUCCESS', - EXPORT_DATASET_FAILED = 'EXPORT_DATASET_FAILED', DELETE_TASK = 'DELETE_TASK', DELETE_TASK_SUCCESS = 'DELETE_TASK_SUCCESS', DELETE_TASK_FAILED = 'DELETE_TASK_FAILED', @@ -108,60 +102,6 @@ export function getTasksAsync(query: TasksQuery): ThunkAction, {}, }; } -function dumpAnnotation(task: any, dumper: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS, - payload: { - task, - dumper, - }, - }; - - return action; -} - -function dumpAnnotationSuccess(task: any, dumper: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS_SUCCESS, - payload: { - task, - dumper, - }, - }; - - return action; -} - -function dumpAnnotationFailed(task: any, dumper: any, error: any): AnyAction { - const action = { - type: TasksActionTypes.DUMP_ANNOTATIONS_FAILED, - payload: { - task, - dumper, - error, - }, - }; - - return action; -} - -export function dumpAnnotationsAsync(task: any, dumper: any): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - try { - dispatch(dumpAnnotation(task, dumper)); - const url = await task.annotations.dump(dumper); - const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; - downloadAnchor.href = url; - downloadAnchor.click(); - } catch (error) { - dispatch(dumpAnnotationFailed(task, dumper, error)); - return; - } - - dispatch(dumpAnnotationSuccess(task, dumper)); - }; -} - function loadAnnotations(task: any, loader: any): AnyAction { const action = { type: TasksActionTypes.LOAD_ANNOTATIONS, @@ -263,60 +203,6 @@ export function importTaskAsync(file: File): ThunkAction, {}, {}, }; } -function exportDataset(task: any, exporter: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET, - payload: { - task, - exporter, - }, - }; - - return action; -} - -function exportDatasetSuccess(task: any, exporter: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET_SUCCESS, - payload: { - task, - exporter, - }, - }; - - return action; -} - -function exportDatasetFailed(task: any, exporter: any, error: any): AnyAction { - const action = { - type: TasksActionTypes.EXPORT_DATASET_FAILED, - payload: { - task, - exporter, - error, - }, - }; - - return action; -} - -export function exportDatasetAsync(task: any, exporter: any): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - dispatch(exportDataset(task, exporter)); - - try { - const url = await task.annotations.exportDataset(exporter.name); - const downloadAnchor = window.document.getElementById('downloadAnchor') as HTMLAnchorElement; - downloadAnchor.href = url; - downloadAnchor.click(); - } catch (error) { - dispatch(exportDatasetFailed(task, exporter, error)); - } - - dispatch(exportDatasetSuccess(task, exporter)); - }; -} - function exportTask(taskID: number): AnyAction { const action = { type: TasksActionTypes.EXPORT_TASK, diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index aa4d2acfd506..d208ef682d09 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -9,9 +9,7 @@ import Modal from 'antd/lib/modal'; import { LoadingOutlined } from '@ant-design/icons'; // eslint-disable-next-line import/no-extraneous-dependencies import { MenuInfo } from 'rc-menu/lib/interface'; -import DumpSubmenu from './dump-submenu'; import LoadSubmenu from './load-submenu'; -import ExportSubmenu from './export-submenu'; import { DimensionType } from '../../reducers/interfaces'; interface Props { @@ -21,8 +19,6 @@ interface Props { loaders: any[]; dumpers: any[]; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; inferenceIsActive: boolean; taskDimension: DimensionType; onClickMenu: (params: MenuInfo, file?: File) => void; @@ -30,7 +26,6 @@ interface Props { } export enum Actions { - DUMP_TASK_ANNO = 'dump_task_anno', LOAD_TASK_ANNO = 'load_task_anno', EXPORT_TASK_DATASET = 'export_task_dataset', DELETE_TASK = 'delete_task', @@ -43,14 +38,10 @@ export enum Actions { export default function ActionsMenuComponent(props: Props): JSX.Element { const { taskID, - taskMode, bugTracker, inferenceIsActive, - dumpers, loaders, onClickMenu, - dumpActivities, - exportActivities, loadActivity, taskDimension, exportIsActive, @@ -106,13 +97,6 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { return ( - {DumpSubmenu({ - taskMode, - dumpers, - dumpActivities, - menuKey: Actions.DUMP_TASK_ANNO, - taskDimension, - })} {LoadSubmenu({ loaders, loadActivity, @@ -122,19 +106,14 @@ export default function ActionsMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_TASK_ANNO, taskDimension, })} - {ExportSubmenu({ - exporters: dumpers, - exportActivities, - menuKey: Actions.EXPORT_TASK_DATASET, - taskDimension, - })} + Export task dataset {!!bugTracker && Open bug tracker} Automatic annotation {exportIsActive && } - Export Task + Export task
Move to project diff --git a/cvat-ui/src/components/actions-menu/dump-submenu.tsx b/cvat-ui/src/components/actions-menu/dump-submenu.tsx deleted file mode 100644 index 91721ac6bb81..000000000000 --- a/cvat-ui/src/components/actions-menu/dump-submenu.tsx +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import Menu from 'antd/lib/menu'; -import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; -import Text from 'antd/lib/typography/Text'; -import { DimensionType } from '../../reducers/interfaces'; - -function isDefaultFormat(dumperName: string, taskMode: string): boolean { - return ( - (dumperName === 'CVAT for video 1.1' && taskMode === 'interpolation') || - (dumperName === 'CVAT for images 1.1' && taskMode === 'annotation') - ); -} - -interface Props { - taskMode: string; - menuKey: string; - dumpers: any[]; - dumpActivities: string[] | null; - taskDimension: DimensionType; -} - -export default function DumpSubmenu(props: Props): JSX.Element { - const { - taskMode, menuKey, dumpers, dumpActivities, taskDimension, - } = props; - - return ( - - {dumpers - .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter((dumper: any): boolean => dumper.dimension === taskDimension) - .map( - (dumper: any): JSX.Element => { - const pending = (dumpActivities || []).includes(dumper.name); - const disabled = !dumper.enabled || pending; - const isDefault = isDefaultFormat(dumper.name, taskMode); - return ( - - - - {dumper.name} - - {pending && } - - ); - }, - )} - - ); -} diff --git a/cvat-ui/src/components/actions-menu/export-submenu.tsx b/cvat-ui/src/components/actions-menu/export-submenu.tsx deleted file mode 100644 index 683565009155..000000000000 --- a/cvat-ui/src/components/actions-menu/export-submenu.tsx +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import Menu from 'antd/lib/menu'; -import Text from 'antd/lib/typography/Text'; -import { ExportOutlined, LoadingOutlined } from '@ant-design/icons'; -import { DimensionType } from '../../reducers/interfaces'; - -interface Props { - menuKey: string; - exporters: any[]; - exportActivities: string[] | null; - taskDimension: DimensionType; -} - -export default function ExportSubmenu(props: Props): JSX.Element { - const { - menuKey, exporters, exportActivities, taskDimension, - } = props; - - return ( - - {exporters - .sort((a: any, b: any) => a.name.localeCompare(b.name)) - .filter((exporter: any): boolean => exporter.dimension === taskDimension) - .map( - (exporter: any): JSX.Element => { - const pending = (exportActivities || []).includes(exporter.name); - const disabled = !exporter.enabled || pending; - return ( - - - {exporter.name} - {pending && } - - ); - }, - )} - - ); -} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/objects-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/objects-side-bar.tsx index 51af99a8187a..302babcdbd93 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/objects-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/objects-side-bar.tsx @@ -60,11 +60,7 @@ function mapDispatchToProps(dispatch: Dispatch): DispatchToProps { function ObjectsSideBar(props: StateToProps & DispatchToProps & OwnProps): JSX.Element { const { - sidebarCollapsed, - canvasInstance, - collapseSidebar, - objectsList, - jobInstance, + sidebarCollapsed, canvasInstance, collapseSidebar, objectsList, jobInstance, } = props; const collapse = (): void => { @@ -119,13 +115,11 @@ function ObjectsSideBar(props: StateToProps & DispatchToProps & OwnProps): JSX.E - {is2D ? - ( - Issues} key='issues'> - - - ) : null} - + {is2D ? ( + Issues} key='issues'> + + + ) : null} {!sidebarCollapsed && } diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index 258cf2b23354..1a9a10164eb0 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -8,9 +8,8 @@ import Modal from 'antd/lib/modal'; // eslint-disable-next-line import/no-extraneous-dependencies import { MenuInfo } from 'rc-menu/lib/interface'; -import DumpSubmenu from 'components/actions-menu/dump-submenu'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import LoadSubmenu from 'components/actions-menu/load-submenu'; -import ExportSubmenu from 'components/actions-menu/export-submenu'; import { DimensionType } from '../../../reducers/interfaces'; interface Props { @@ -18,8 +17,6 @@ interface Props { loaders: any[]; dumpers: any[]; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; isReviewer: boolean; jobInstance: any; onClickMenu(params: MenuInfo, file?: File): void; @@ -28,7 +25,6 @@ interface Props { } export enum Actions { - DUMP_TASK_ANNO = 'dump_task_anno', LOAD_JOB_ANNO = 'load_job_anno', EXPORT_TASK_DATASET = 'export_task_dataset', REMOVE_ANNO = 'remove_anno', @@ -41,12 +37,8 @@ export enum Actions { export default function AnnotationMenuComponent(props: Props): JSX.Element { const { - taskMode, loaders, - dumpers, loadActivity, - dumpActivities, - exportActivities, isReviewer, jobInstance, onClickMenu, @@ -163,13 +155,6 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { return ( - {DumpSubmenu({ - taskMode, - dumpers, - dumpActivities, - menuKey: Actions.DUMP_TASK_ANNO, - taskDimension: jobInstance.task.dimension, - })} {LoadSubmenu({ loaders, loadActivity, @@ -179,13 +164,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { menuKey: Actions.LOAD_JOB_ANNO, taskDimension: jobInstance.task.dimension, })} - {ExportSubmenu({ - exporters: dumpers, - exportActivities, - menuKey: Actions.EXPORT_TASK_DATASET, - taskDimension: jobInstance.task.dimension, - })} - + Export task dataset Remove annotations e.preventDefault()}> @@ -198,6 +177,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { Submit the review )} {jobStatus === 'completed' && Renew the job} + ); } diff --git a/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx new file mode 100644 index 000000000000..400cdc4e21c7 --- /dev/null +++ b/cvat-ui/src/components/export-dataset/export-dataset-modal.tsx @@ -0,0 +1,145 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import './styles.scss'; +import React, { useState, useEffect, useCallback } from 'react'; +import Modal from 'antd/lib/modal'; +import Notification from 'antd/lib/notification'; +import { useSelector, useDispatch } from 'react-redux'; +import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons'; +import Text from 'antd/lib/typography/Text'; +import Select from 'antd/lib/select'; +import Checkbox from 'antd/lib/checkbox'; +import Input from 'antd/lib/input'; +import Form from 'antd/lib/form'; + +import { CombinedState } from 'reducers/interfaces'; +import { exportActions, exportDatasetAsync } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; + +const core = getCore(); + +type FormValues = { + selectedFormat: string | undefined; + saveImages: boolean; + customName: string | undefined; +}; + +function ExportDatasetModal(): JSX.Element { + const [instanceType, setInstanceType] = useState(''); + const [activities, setActivities] = useState([]); + const [form] = Form.useForm(); + const dispatch = useDispatch(); + const instance = useSelector((state: CombinedState) => state.export.instance); + const modalVisible = useSelector((state: CombinedState) => state.export.modalVisible); + const dumpers = useSelector((state: CombinedState) => state.formats.annotationFormats.dumpers); + const { + tasks: taskExportActivities, projects: projectExportActivities, + } = useSelector((state: CombinedState) => state.export); + + const initActivities = (): void => { + if (instance instanceof core.classes.Project) { + setInstanceType('project'); + setActivities(projectExportActivities[instance.id] || []); + } else if (instance instanceof core.classes.Task) { + setInstanceType('task'); + setActivities(taskExportActivities[instance.id] || []); + if (instance.mode === 'interpolation' && instance.dimension === '2d') { + form.setFieldsValue({ selectedFormat: 'CVAT for video 1.1' }); + } else if (instance.mode === 'annotation' && instance.dimension === '2d') { + form.setFieldsValue({ selectedFormat: 'CVAT for images 1.1' }); + } + } + }; + + useEffect(() => { + initActivities(); + }, [instance?.id, instance instanceof core.classes.Project]); + + const closeModal = (): void => { + form.resetFields(); + dispatch(exportActions.closeExportModal()); + }; + + const handleExport = useCallback((values: FormValues): void => { + // have to validate format before so it would not be undefined + dispatch( + exportDatasetAsync(instance, values.selectedFormat as string, values.customName ? `${values.customName}.zip` : '', values.saveImages), + ); + closeModal(); + Notification.info({ + message: 'Dataset export started', + description: `Dataset export was started for ${instanceType} #${instance?.id}. ` + + 'Download will start automaticly as soon as the dataset is ready.', + className: `cvat-notification-notice-export-${instanceType}-start`, + }); + }, [instance?.id, instance instanceof core.classes.Project, instanceType]); + + return ( + form.submit()} + className={`cvat-modal-export-${instanceType}`} + > +
+ + + + + Save images + + + + +
+ + ); +} + +export default React.memo(ExportDatasetModal); diff --git a/cvat-ui/src/components/export-dataset/styles.scss b/cvat-ui/src/components/export-dataset/styles.scss new file mode 100644 index 000000000000..26946bd0f8f6 --- /dev/null +++ b/cvat-ui/src/components/export-dataset/styles.scss @@ -0,0 +1,13 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +@import '../../base.scss'; + +.cvat-modal-export-option-item > .ant-select-item-option-content, +.cvat-modal-export-select .ant-select-selection-item { + > span[role='img'] { + color: $info-icon-color; + margin-right: $grid-unit-size; + } +} diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx index ef44d03122cf..b2bb7a62aeeb 100644 --- a/cvat-ui/src/components/project-page/project-page.tsx +++ b/cvat-ui/src/components/project-page/project-page.tsx @@ -16,6 +16,7 @@ import { PlusOutlined } from '@ant-design/icons'; import { CombinedState, Task } from 'reducers/interfaces'; import { getProjectsAsync } from 'actions/projects-actions'; import { cancelInferenceAsync } from 'actions/models-actions'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import TaskItem from 'components/tasks-page/task-item'; import MoveTaskModal from 'components/move-task-modal/move-task-modal'; import ModelRunnerDialog from 'components/model-runner-modal/model-runner-dialog'; @@ -111,6 +112,7 @@ export default function ProjectPageComponent(): JSX.Element { ))} + diff --git a/cvat-ui/src/components/projects-page/actions-menu.tsx b/cvat-ui/src/components/projects-page/actions-menu.tsx index 35fefea5ef18..75d71508652f 100644 --- a/cvat-ui/src/components/projects-page/actions-menu.tsx +++ b/cvat-ui/src/components/projects-page/actions-menu.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -8,6 +8,7 @@ import Modal from 'antd/lib/modal'; import Menu from 'antd/lib/menu'; import { deleteProjectAsync } from 'actions/projects-actions'; +import { exportActions } from 'actions/export-actions'; interface Props { projectInstance: any; @@ -37,6 +38,11 @@ export default function ProjectActionsMenuComponent(props: Props): JSX.Element { return ( Delete + dispatch(exportActions.openExportModal(projectInstance))} + > + Export project dataset + ); } diff --git a/cvat-ui/src/components/projects-page/projects-page.tsx b/cvat-ui/src/components/projects-page/projects-page.tsx index 5d4d129a9af9..a2842348af50 100644 --- a/cvat-ui/src/components/projects-page/projects-page.tsx +++ b/cvat-ui/src/components/projects-page/projects-page.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -8,9 +8,10 @@ import { useDispatch, useSelector } from 'react-redux'; import { useLocation, useHistory } from 'react-router'; import Spin from 'antd/lib/spin'; -import FeedbackComponent from 'components/feedback/feedback'; import { CombinedState, ProjectsQuery } from 'reducers/interfaces'; import { getProjectsAsync } from 'actions/projects-actions'; +import FeedbackComponent from 'components/feedback/feedback'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import EmptyListComponent from './empty-list'; import TopBarComponent from './top-bar'; import ProjectListComponent from './project-list'; @@ -55,6 +56,7 @@ export default function ProjectsPageComponent(): JSX.Element { {projectsCount ? : } + ); } diff --git a/cvat-ui/src/components/task-page/task-page.tsx b/cvat-ui/src/components/task-page/task-page.tsx index 627dbf9d2504..0465fa95d52e 100644 --- a/cvat-ui/src/components/task-page/task-page.tsx +++ b/cvat-ui/src/components/task-page/task-page.tsx @@ -14,6 +14,7 @@ import DetailsContainer from 'containers/task-page/details'; import JobListContainer from 'containers/task-page/job-list'; import ModelRunnerModal from 'components/model-runner-modal/model-runner-dialog'; import MoveTaskModal from 'components/move-task-modal/move-task-modal'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import { Task } from 'reducers/interfaces'; import TopBarComponent from './top-bar'; @@ -85,6 +86,7 @@ class TaskPageComponent extends React.PureComponent { + {updating && } ); diff --git a/cvat-ui/src/components/tasks-page/tasks-page.tsx b/cvat-ui/src/components/tasks-page/tasks-page.tsx index 6e0df457716e..1420822b1e40 100644 --- a/cvat-ui/src/components/tasks-page/tasks-page.tsx +++ b/cvat-ui/src/components/tasks-page/tasks-page.tsx @@ -14,6 +14,7 @@ import Text from 'antd/lib/typography/Text'; import { TasksQuery } from 'reducers/interfaces'; import FeedbackComponent from 'components/feedback/feedback'; import TaskListContainer from 'containers/tasks-page/tasks-list'; +import ExportDatasetModal from 'components/export-dataset/export-dataset-modal'; import TopBar from './top-bar'; import EmptyListComponent from './empty-list'; @@ -221,6 +222,7 @@ class TasksPageComponent extends React.PureComponent )} + ); } diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index 5923928c0a59..5c38825cb107 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -12,13 +12,12 @@ import { CombinedState } from 'reducers/interfaces'; import { modelsActions } from 'actions/models-actions'; import { - dumpAnnotationsAsync, loadAnnotationsAsync, - exportDatasetAsync, deleteTaskAsync, exportTaskAsync, switchMoveTaskModalVisible, } from 'actions/tasks-actions'; +import { exportActions } from 'actions/export-actions'; interface OwnProps { taskInstance: any; @@ -27,16 +26,13 @@ interface OwnProps { interface StateToProps { annotationFormats: any; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; inferenceIsActive: boolean; exportIsActive: boolean; } interface DispatchToProps { loadAnnotations: (taskInstance: any, loader: any, file: File) => void; - dumpAnnotations: (taskInstance: any, dumper: any) => void; - exportDataset: (taskInstance: any, exporter: any) => void; + showExportModal: (taskInstance: any) => void; deleteTask: (taskInstance: any) => void; openRunModelWindow: (taskInstance: any) => void; exportTask: (taskInstance: any) => void; @@ -52,14 +48,12 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { formats: { annotationFormats }, tasks: { activities: { - dumps, loads, exports: activeExports, backups, + loads, backups, }, }, } = state; return { - dumpActivities: tid in dumps ? dumps[tid] : null, - exportActivities: tid in activeExports ? activeExports[tid] : null, loadActivity: tid in loads ? loads[tid] : null, annotationFormats, inferenceIsActive: tid in state.models.inferences, @@ -72,11 +66,8 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { loadAnnotations: (taskInstance: any, loader: any, file: File): void => { dispatch(loadAnnotationsAsync(taskInstance, loader, file)); }, - dumpAnnotations: (taskInstance: any, dumper: any): void => { - dispatch(dumpAnnotationsAsync(taskInstance, dumper)); - }, - exportDataset: (taskInstance: any, exporter: any): void => { - dispatch(exportDatasetAsync(taskInstance, exporter)); + showExportModal: (taskInstance: any): void => { + dispatch(exportActions.openExportModal(taskInstance)); }, deleteTask: (taskInstance: any): void => { dispatch(deleteTaskAsync(taskInstance)); @@ -98,14 +89,11 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): taskInstance, annotationFormats: { loaders, dumpers }, loadActivity, - dumpActivities, - exportActivities, inferenceIsActive, exportIsActive, loadAnnotations, - dumpAnnotations, - exportDataset, + showExportModal, deleteTask, openRunModelWindow, exportTask, @@ -115,28 +103,18 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): function onClickMenu(params: MenuInfo, file?: File): void { if (params.keyPath.length > 1) { const [additionalKey, action] = params.keyPath; - if (action === Actions.DUMP_TASK_ANNO) { - const format = additionalKey; - const [dumper] = dumpers.filter((_dumper: any): boolean => _dumper.name === format); - if (dumper) { - dumpAnnotations(taskInstance, dumper); - } - } else if (action === Actions.LOAD_TASK_ANNO) { + if (action === Actions.LOAD_TASK_ANNO) { const format = additionalKey; const [loader] = loaders.filter((_loader: any): boolean => _loader.name === format); if (loader && file) { loadAnnotations(taskInstance, loader, file); } - } else if (action === Actions.EXPORT_TASK_DATASET) { - const format = additionalKey; - const [exporter] = dumpers.filter((_exporter: any): boolean => _exporter.name === format); - if (exporter) { - exportDataset(taskInstance, exporter); - } } } else { const [action] = params.keyPath; - if (action === Actions.DELETE_TASK) { + if (action === Actions.EXPORT_TASK_DATASET) { + showExportModal(taskInstance); + } else if (action === Actions.DELETE_TASK) { deleteTask(taskInstance); } else if (action === Actions.OPEN_BUG_TRACKER) { window.open(`${taskInstance.bugTracker}`, '_blank'); @@ -158,8 +136,6 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): loaders={loaders} dumpers={dumpers} loadActivity={loadActivity} - dumpActivities={dumpActivities} - exportActivities={exportActivities} inferenceIsActive={inferenceIsActive} onClickMenu={onClickMenu} taskDimension={taskInstance.dimension} diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index a52d3e11a0f2..60d5fef6fa6f 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -10,7 +10,7 @@ import { MenuInfo } from 'rc-menu/lib/interface'; import { CombinedState, TaskStatus } from 'reducers/interfaces'; import AnnotationMenuComponent, { Actions } from 'components/annotation-page/top-bar/annotation-menu'; -import { dumpAnnotationsAsync, exportDatasetAsync, updateJobAsync } from 'actions/tasks-actions'; +import { updateJobAsync } from 'actions/tasks-actions'; import { uploadJobAnnotationsAsync, removeAnnotationsAsync, @@ -19,20 +19,18 @@ import { switchSubmitReviewDialog as switchSubmitReviewDialogAction, setForceExitAnnotationFlag as setForceExitAnnotationFlagAction, } from 'actions/annotation-actions'; +import { exportActions } from 'actions/export-actions'; interface StateToProps { annotationFormats: any; jobInstance: any; loadActivity: string | null; - dumpActivities: string[] | null; - exportActivities: string[] | null; user: any; } interface DispatchToProps { loadAnnotations(job: any, loader: any, file: File): void; - dumpAnnotations(task: any, dumper: any): void; - exportDataset(task: any, exporter: any): void; + showExportModal(task: any): void; removeAnnotations(sessionInstance: any): void; switchRequestReviewDialog(visible: boolean): void; switchSubmitReviewDialog(visible: boolean): void; @@ -49,7 +47,7 @@ function mapStateToProps(state: CombinedState): StateToProps { }, formats: { annotationFormats }, tasks: { - activities: { dumps, loads, exports: activeExports }, + activities: { loads }, }, auth: { user }, } = state; @@ -58,8 +56,6 @@ function mapStateToProps(state: CombinedState): StateToProps { const jobID = jobInstance.id; return { - dumpActivities: taskID in dumps ? dumps[taskID] : null, - exportActivities: taskID in activeExports ? activeExports[taskID] : null, loadActivity: taskID in loads || jobID in jobLoads ? loads[taskID] || jobLoads[jobID] : null, jobInstance, annotationFormats, @@ -72,11 +68,8 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { loadAnnotations(job: any, loader: any, file: File): void { dispatch(uploadJobAnnotationsAsync(job, loader, file)); }, - dumpAnnotations(task: any, dumper: any): void { - dispatch(dumpAnnotationsAsync(task, dumper)); - }, - exportDataset(task: any, exporter: any): void { - dispatch(exportDatasetAsync(task, exporter)); + showExportModal(task: any): void { + dispatch(exportActions.openExportModal(task)); }, removeAnnotations(sessionInstance: any): void { dispatch(removeAnnotationsAsync(sessionInstance)); @@ -108,11 +101,8 @@ function AnnotationMenuContainer(props: Props): JSX.Element { annotationFormats: { loaders, dumpers }, history, loadActivity, - dumpActivities, - exportActivities, loadAnnotations, - dumpAnnotations, - exportDataset, + showExportModal, removeAnnotations, switchRequestReviewDialog, switchSubmitReviewDialog, @@ -124,28 +114,18 @@ function AnnotationMenuContainer(props: Props): JSX.Element { const onClickMenu = (params: MenuInfo, file?: File): void => { if (params.keyPath.length > 1) { const [additionalKey, action] = params.keyPath; - if (action === Actions.DUMP_TASK_ANNO) { - const format = additionalKey; - const [dumper] = dumpers.filter((_dumper: any): boolean => _dumper.name === format); - if (dumper) { - dumpAnnotations(jobInstance.task, dumper); - } - } else if (action === Actions.LOAD_JOB_ANNO) { + if (action === Actions.LOAD_JOB_ANNO) { const format = additionalKey; const [loader] = loaders.filter((_loader: any): boolean => _loader.name === format); if (loader && file) { loadAnnotations(jobInstance, loader, file); } - } else if (action === Actions.EXPORT_TASK_DATASET) { - const format = additionalKey; - const [exporter] = dumpers.filter((_exporter: any): boolean => _exporter.name === format); - if (exporter) { - exportDataset(jobInstance.task, exporter); - } } } else { const [action] = params.keyPath; - if (action === Actions.REMOVE_ANNO) { + if (action === Actions.EXPORT_TASK_DATASET) { + showExportModal(jobInstance.task); + } else if (action === Actions.REMOVE_ANNO) { removeAnnotations(jobInstance); } else if (action === Actions.REQUEST_REVIEW) { switchRequestReviewDialog(true); @@ -173,8 +153,6 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loaders={loaders} dumpers={dumpers} loadActivity={loadActivity} - dumpActivities={dumpActivities} - exportActivities={exportActivities} onClickMenu={onClickMenu} setForceExitAnnotationFlag={setForceExitAnnotationFlag} saveAnnotations={saveAnnotations} diff --git a/cvat-ui/src/reducers/export-reducer.ts b/cvat-ui/src/reducers/export-reducer.ts new file mode 100644 index 000000000000..a7a023524f0d --- /dev/null +++ b/cvat-ui/src/reducers/export-reducer.ts @@ -0,0 +1,67 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import { ExportActions, ExportActionTypes } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; +import deepCopy from 'utils/deep-copy'; + +import { ExportState } from './interfaces'; + +const core = getCore(); + +const defaultState: ExportState = { + tasks: {}, + projects: {}, + instance: null, + modalVisible: false, +}; + +export default (state: ExportState = defaultState, action: ExportActions): ExportState => { + switch (action.type) { + case ExportActionTypes.OPEN_EXPORT_MODAL: + return { + ...state, + modalVisible: true, + instance: action.payload.instance, + }; + case ExportActionTypes.CLOSE_EXPORT_MODAL: + return { + ...state, + modalVisible: false, + instance: null, + }; + case ExportActionTypes.EXPORT_DATASET: { + const { instance, format } = action.payload; + const activities = deepCopy(instance instanceof core.classes.Project ? state.projects : state.tasks); + + activities[instance.id] = + instance.id in activities && !activities[instance.id].includes(format) ? + [...activities[instance.id], format] : + activities[instance.id] || [format]; + + return { + ...state, + tasks: instance instanceof core.classes.Task ? activities : state.tasks, + projects: instance instanceof core.classes.Project ? activities : state.projects, + }; + } + case ExportActionTypes.EXPORT_DATASET_FAILED: + case ExportActionTypes.EXPORT_DATASET_SUCCESS: { + const { instance, format } = action.payload; + const activities = deepCopy(instance instanceof core.classes.Project ? state.projects : state.tasks); + + activities[instance.id] = activities[instance.id].filter( + (exporterName: string): boolean => exporterName !== format, + ); + + return { + ...state, + tasks: instance instanceof core.classes.Task ? activities : state.tasks, + projects: instance instanceof core.classes.Project ? activities : state.projects, + }; + } + default: + return state; + } +}; diff --git a/cvat-ui/src/reducers/formats-reducer.ts b/cvat-ui/src/reducers/formats-reducer.ts index 855caaa53df9..8260e976ad56 100644 --- a/cvat-ui/src/reducers/formats-reducer.ts +++ b/cvat-ui/src/reducers/formats-reducer.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index adbe7bffcdf5..f89e268d5453 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -86,14 +86,6 @@ export interface TasksState { count: number; current: Task[]; activities: { - dumps: { - // dumps in different formats at the same time - [tid: number]: string[]; // dumper names - }; - exports: { - // exports in different formats at the same time - [tid: number]: string[]; // dumper names - }; loads: { // only one loading simultaneously [tid: number]: string; // loader name @@ -112,6 +104,17 @@ export interface TasksState { }; } +export interface ExportState { + tasks: { + [tid: number]: string[]; + }; + projects: { + [pid: number]: string[]; + }; + instance: any; + modalVisible: boolean; +} + export interface FormatsState { annotationFormats: any; fetching: boolean; @@ -621,6 +624,7 @@ export interface CombinedState { settings: SettingsState; shortcuts: ShortcutsState; review: ReviewState; + export: ExportState; } export enum DimensionType { diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index e09db0b2cd6a..34907060ed70 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -16,9 +16,13 @@ import { NotificationsActionType } from 'actions/notification-actions'; import { BoundariesActionTypes } from 'actions/boundaries-actions'; import { UserAgreementsActionTypes } from 'actions/useragreements-actions'; import { ReviewActionTypes } from 'actions/review-actions'; +import { ExportActionTypes } from 'actions/export-actions'; +import getCore from 'cvat-core-wrapper'; import { NotificationsState } from './interfaces'; +const core = getCore(); + const defaultState: NotificationsState = { errors: { auth: { @@ -308,8 +312,9 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case TasksActionTypes.EXPORT_DATASET_FAILED: { - const taskID = action.payload.task.id; + case ExportActionTypes.EXPORT_DATASET_FAILED: { + const instanceID = action.payload.instance.id; + const instanceType = action.payload.instance instanceof core.classes.Project ? 'project' : 'task'; return { ...state, errors: { @@ -319,7 +324,8 @@ export default function (state = defaultState, action: AnyAction): Notifications exportingAsDataset: { message: 'Could not export dataset for the ' + - `task ${taskID}`, + `` + + `${instanceType} ${instanceID}`, reason: action.payload.error.toString(), }, }, @@ -392,24 +398,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case TasksActionTypes.DUMP_ANNOTATIONS_FAILED: { - const taskID = action.payload.task.id; - return { - ...state, - errors: { - ...state.errors, - tasks: { - ...state.errors.tasks, - dumping: { - message: - 'Could not dump annotations for the ' + - `task ${taskID}`, - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case TasksActionTypes.DELETE_TASK_FAILED: { const { taskID } = action.payload; return { diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index 04358b44e636..b1219b7a0b23 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -17,6 +17,7 @@ import settingsReducer from './settings-reducer'; import shortcutsReducer from './shortcuts-reducer'; import userAgreementsReducer from './useragreements-reducer'; import reviewReducer from './review-reducer'; +import exportReducer from './export-reducer'; export default function createRootReducer(): Reducer { return combineReducers({ @@ -34,5 +35,6 @@ export default function createRootReducer(): Reducer { shortcuts: shortcutsReducer, userAgreements: userAgreementsReducer, review: reviewReducer, + export: exportReducer, }); } diff --git a/cvat-ui/src/reducers/tasks-reducer.ts b/cvat-ui/src/reducers/tasks-reducer.ts index 78236132a9e3..592286d16503 100644 --- a/cvat-ui/src/reducers/tasks-reducer.ts +++ b/cvat-ui/src/reducers/tasks-reducer.ts @@ -32,8 +32,6 @@ const defaultState: TasksState = { mode: null, }, activities: { - dumps: {}, - exports: {}, loads: {}, deletes: {}, creates: { @@ -85,84 +83,6 @@ export default (state: TasksState = defaultState, action: AnyAction): TasksState initialized: true, fetching: false, }; - case TasksActionTypes.DUMP_ANNOTATIONS: { - const { task } = action.payload; - const { dumper } = action.payload; - const { dumps } = state.activities; - - dumps[task.id] = - task.id in dumps && !dumps[task.id].includes(dumper.name) ? - [...dumps[task.id], dumper.name] : - dumps[task.id] || [dumper.name]; - - return { - ...state, - activities: { - ...state.activities, - dumps: { - ...dumps, - }, - }, - }; - } - case TasksActionTypes.DUMP_ANNOTATIONS_FAILED: - case TasksActionTypes.DUMP_ANNOTATIONS_SUCCESS: { - const { task } = action.payload; - const { dumper } = action.payload; - const { dumps } = state.activities; - - dumps[task.id] = dumps[task.id].filter((dumperName: string): boolean => dumperName !== dumper.name); - - return { - ...state, - activities: { - ...state.activities, - dumps: { - ...dumps, - }, - }, - }; - } - case TasksActionTypes.EXPORT_DATASET: { - const { task } = action.payload; - const { exporter } = action.payload; - const { exports: activeExports } = state.activities; - - activeExports[task.id] = - task.id in activeExports && !activeExports[task.id].includes(exporter.name) ? - [...activeExports[task.id], exporter.name] : - activeExports[task.id] || [exporter.name]; - - return { - ...state, - activities: { - ...state.activities, - exports: { - ...activeExports, - }, - }, - }; - } - case TasksActionTypes.EXPORT_DATASET_FAILED: - case TasksActionTypes.EXPORT_DATASET_SUCCESS: { - const { task } = action.payload; - const { exporter } = action.payload; - const { exports: activeExports } = state.activities; - - activeExports[task.id] = activeExports[task.id].filter( - (exporterName: string): boolean => exporterName !== exporter.name, - ); - - return { - ...state, - activities: { - ...state.activities, - exports: { - ...activeExports, - }, - }, - }; - } case TasksActionTypes.LOAD_ANNOTATIONS: { const { task } = action.payload; const { loader } = action.payload; diff --git a/cvat-ui/src/utils/deep-copy.ts b/cvat-ui/src/utils/deep-copy.ts new file mode 100644 index 000000000000..986c4dbea5b3 --- /dev/null +++ b/cvat-ui/src/utils/deep-copy.ts @@ -0,0 +1,21 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +function deepCopy(obj: T): T { + if (typeof obj !== 'object') { + return obj; + } + if (!obj) { + return obj; + } + const container: any = (obj instanceof Array) ? [] : {}; + for (const i in obj) { + if (Object.prototype.hasOwnProperty.call(obj, i)) { + container[i] = deepCopy(obj[i]); + } + } + return container; +} + +export default deepCopy; diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index 0fbd6bf09f5e..88989dbb4921 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -42,6 +42,9 @@ def data(self): def __getitem__(self, key): return getattr(self, key) + def __setitem__(self, key, value): + return setattr(self, key, value) + @data.setter def data(self, data): self.version = data['version'] diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 82716123b642..6a61b2cf93cf 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -3,52 +3,34 @@ # # SPDX-License-Identifier: MIT +import sys import os.path as osp -from collections import OrderedDict, namedtuple +from collections import namedtuple +from typing import Any, Callable, DefaultDict, Dict, List, Literal, Mapping, NamedTuple, OrderedDict, Tuple, Union from pathlib import Path from django.utils import timezone import datumaro.components.extractor as datumaro from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import AttributeType, ShapeType, DimensionType, Image as Img +from cvat.apps.engine.models import AttributeType, ShapeType, Project, Task, Label, DimensionType, Image as Img from datumaro.util import cast from datumaro.util.image import ByteImage, Image -from .annotation import AnnotationManager, TrackManager +from .annotation import AnnotationManager, TrackManager, AnnotationIR -class TaskData: - Attribute = namedtuple('Attribute', 'name, value') - Shape = namedtuple("Shape", 'id, label_id') # 3d - LabeledShape = namedtuple( - 'LabeledShape', 'type, frame, label, points, occluded, attributes, source, group, z_order') - LabeledShape.__new__.__defaults__ = (0, 0) - TrackedShape = namedtuple( - 'TrackedShape', 'type, frame, points, occluded, outside, keyframe, attributes, source, group, z_order, label, track_id') - TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) - Track = namedtuple('Track', 'label, group, source, shapes') - Tag = namedtuple('Tag', 'frame, label, attributes, source, group') - Tag.__new__.__defaults__ = (0, ) - Frame = namedtuple( - 'Frame', 'idx, id, frame, name, width, height, labeled_shapes, tags, shapes, labels') - Labels = namedtuple('Label', 'id, name, color') +class InstanceLabelData: + Attribute = NamedTuple('Attribute', [('name', str), ('value', Any)]) - def __init__(self, annotation_ir, db_task, host='', create_callback=None): - self._annotation_ir = annotation_ir - self._db_task = db_task - self._host = host - self._create_callback = create_callback - self._MAX_ANNO_SIZE = 30000 - self._frame_info = {} - self._frame_mapping = {} - self._frame_step = db_task.data.get_frame_step() + def __init__(self, instance: Union[Task, Project]) -> None: + instance = instance.project if isinstance(instance, Task) and instance.project_id is not None else instance - db_labels = (self._db_task.project if self._db_task.project_id else self._db_task).label_set.all().prefetch_related( - 'attributespec_set').order_by('pk') + db_labels = instance.label_set.all().prefetch_related('attributespec_set').order_by('pk') - self._label_mapping = OrderedDict( - (db_label.id, db_label) for db_label in db_labels) + self._label_mapping = OrderedDict[int, Label]( + ((db_label.id, db_label) for db_label in db_labels), + ) self._attribute_mapping = {db_label.id: { 'mutable': {}, 'immutable': {}, 'spec': {}} @@ -69,9 +51,6 @@ def __init__(self, annotation_ir, db_task, host='', create_callback=None): **attr_mapping['immutable'], } - self._init_frame_info() - self._init_meta() - def _get_label_id(self, label_name): for db_label in self._label_mapping.values(): if label_name == db_label.name: @@ -103,6 +82,71 @@ def _get_mutable_attribute_id(self, label_id, attribute_name): def _get_immutable_attribute_id(self, label_id, attribute_name): return self._get_attribute_id(label_id, attribute_name, 'immutable') + def _import_attribute(self, label_id, attribute): + spec_id = self._get_attribute_id(label_id, attribute.name) + value = attribute.value + + if spec_id: + spec = self._attribute_mapping[label_id]['spec'][spec_id] + + try: + if spec.input_type == AttributeType.NUMBER: + pass # no extra processing required + elif spec.input_type == AttributeType.CHECKBOX: + if isinstance(value, str): + value = value.lower() + assert value in {'true', 'false'} + elif isinstance(value, (bool, int, float)): + value = 'true' if value else 'false' + else: + raise ValueError("Unexpected attribute value") + except Exception as e: + raise Exception("Failed to convert attribute '%s'='%s': %s" % + (self._get_label_name(label_id), value, e)) + + return { 'spec_id': spec_id, 'value': value } + + def _export_attributes(self, attributes): + exported_attributes = [] + for attr in attributes: + attribute_name = self._get_attribute_name(attr["spec_id"]) + exported_attributes.append(InstanceLabelData.Attribute( + name=attribute_name, + value=attr["value"], + )) + return exported_attributes + + +class TaskData(InstanceLabelData): + Shape = namedtuple("Shape", 'id, label_id') # 3d + LabeledShape = namedtuple( + 'LabeledShape', 'type, frame, label, points, occluded, attributes, source, group, z_order') + LabeledShape.__new__.__defaults__ = (0, 0) + TrackedShape = namedtuple( + 'TrackedShape', 'type, frame, points, occluded, outside, keyframe, attributes, source, group, z_order, label, track_id') + TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) + Track = namedtuple('Track', 'label, group, source, shapes') + Tag = namedtuple('Tag', 'frame, label, attributes, source, group') + Tag.__new__.__defaults__ = (0, ) + Frame = namedtuple( + 'Frame', 'idx, id, frame, name, width, height, labeled_shapes, tags, shapes, labels') + Labels = namedtuple('Label', 'id, name, color') + + def __init__(self, annotation_ir, db_task, host='', create_callback=None): + self._annotation_ir = annotation_ir + self._db_task = db_task + self._host = host + self._create_callback = create_callback + self._MAX_ANNO_SIZE = 30000 + self._frame_info = {} + self._frame_mapping = {} + self._frame_step = db_task.data.get_frame_step() + + InstanceLabelData.__init__(self, db_task) + + self._init_frame_info() + self._init_meta() + def abs_frame_id(self, relative_id): if relative_id not in range(0, self._db_task.data.size): raise ValueError("Unknown internal frame id %s" % relative_id) @@ -135,79 +179,80 @@ def _init_frame_info(self): for frame_number, info in self._frame_info.items() } - def _init_meta(self): - db_segments = self._db_task.segment_set.all().prefetch_related('job_set') - self._meta = OrderedDict([ - ("task", OrderedDict([ - ("id", str(self._db_task.id)), - ("name", self._db_task.name), - ("size", str(self._db_task.data.size)), - ("mode", self._db_task.mode), - ("overlap", str(self._db_task.overlap)), - ("bugtracker", self._db_task.bug_tracker), - ("created", str(timezone.localtime(self._db_task.created_date))), - ("updated", str(timezone.localtime(self._db_task.updated_date))), - ("start_frame", str(self._db_task.data.start_frame)), - ("stop_frame", str(self._db_task.data.stop_frame)), - ("frame_filter", self._db_task.data.frame_filter), - - ("labels", [ - ("label", OrderedDict([ - ("name", db_label.name), - ("color", db_label.color), - ("attributes", [ - ("attribute", OrderedDict([ - ("name", db_attr.name), - ("mutable", str(db_attr.mutable)), - ("input_type", db_attr.input_type), - ("default_value", db_attr.default_value), - ("values", db_attr.values)])) - for db_attr in db_label.attributespec_set.all()]) - ])) for db_label in self._label_mapping.values() - ]), + @staticmethod + def meta_for_task(db_task, host, label_mapping=None): + db_segments = db_task.segment_set.all().prefetch_related('job_set') + + meta = OrderedDict([ + ("id", str(db_task.id)), + ("name", db_task.name), + ("size", str(db_task.data.size)), + ("mode", db_task.mode), + ("overlap", str(db_task.overlap)), + ("bugtracker", db_task.bug_tracker), + ("created", str(timezone.localtime(db_task.created_date))), + ("updated", str(timezone.localtime(db_task.updated_date))), + ("subset", db_task.subset or datumaro.DEFAULT_SUBSET_NAME), + ("start_frame", str(db_task.data.start_frame)), + ("stop_frame", str(db_task.data.stop_frame)), + ("frame_filter", db_task.data.frame_filter), + + ("segments", [ + ("segment", OrderedDict([ + ("id", str(db_segment.id)), + ("start", str(db_segment.start_frame)), + ("stop", str(db_segment.stop_frame)), + ("url", "{}/?id={}".format( + host, db_segment.job_set.all()[0].id))] + )) for db_segment in db_segments + ]), + + ("owner", OrderedDict([ + ("username", db_task.owner.username), + ("email", db_task.owner.email) + ]) if db_task.owner else ""), + + ("assignee", OrderedDict([ + ("username", db_task.assignee.username), + ("email", db_task.assignee.email) + ]) if db_task.assignee else ""), + ]) - ("segments", [ - ("segment", OrderedDict([ - ("id", str(db_segment.id)), - ("start", str(db_segment.start_frame)), - ("stop", str(db_segment.stop_frame)), - ("url", "{}/?id={}".format( - self._host, db_segment.job_set.all()[0].id))] - )) for db_segment in db_segments - ]), + if label_mapping is not None: + meta['labels'] = [ + ("label", OrderedDict([ + ("name", db_label.name), + ("color", db_label.color), + ("attributes", [ + ("attribute", OrderedDict([ + ("name", db_attr.name), + ("mutable", str(db_attr.mutable)), + ("input_type", db_attr.input_type), + ("default_value", db_attr.default_value), + ("values", db_attr.values)])) + for db_attr in db_label.attributespec_set.all()]) + ])) for db_label in label_mapping.values() + ] + + if hasattr(db_task.data, "video"): + meta["original_size"] = OrderedDict([ + ("width", str(db_task.data.video.width)), + ("height", str(db_task.data.video.height)) + ]) - ("owner", OrderedDict([ - ("username", self._db_task.owner.username), - ("email", self._db_task.owner.email) - ]) if self._db_task.owner else ""), + return meta - ("assignee", OrderedDict([ - ("username", self._db_task.assignee.username), - ("email", self._db_task.assignee.email) - ]) if self._db_task.assignee else ""), - ])), + def _init_meta(self): + self._meta = OrderedDict([ + ("task", self.meta_for_task(self._db_task, self._host, self._label_mapping)), ("dumped", str(timezone.localtime(timezone.now()))) ]) if hasattr(self._db_task.data, "video"): - self._meta["task"]["original_size"] = OrderedDict([ - ("width", str(self._db_task.data.video.width)), - ("height", str(self._db_task.data.video.height)) - ]) # Add source to dumped file self._meta["source"] = str( osp.basename(self._db_task.data.video.path)) - def _export_attributes(self, attributes): - exported_attributes = [] - for attr in attributes: - attribute_name = self._get_attribute_name(attr["spec_id"]) - exported_attributes.append(TaskData.Attribute( - name=attribute_name, - value=attr["value"], - )) - return exported_attributes - def _export_tracked_shape(self, shape): return TaskData.TrackedShape( type=shape["type"], @@ -356,30 +401,6 @@ def _import_tag(self, tag): if self._get_attribute_id(label_id, attrib.name)] return _tag - def _import_attribute(self, label_id, attribute): - spec_id = self._get_attribute_id(label_id, attribute.name) - value = attribute.value - - if spec_id: - spec = self._attribute_mapping[label_id]['spec'][spec_id] - - try: - if spec.input_type == AttributeType.NUMBER: - pass # no extra processing required - elif spec.input_type == AttributeType.CHECKBOX: - if isinstance(value, str): - value = value.lower() - assert value in {'true', 'false'} - elif isinstance(value, (bool, int, float)): - value = 'true' if value else 'false' - else: - raise ValueError("Unexpected attribute value") - except Exception as e: - raise Exception("Failed to convert attribute '%s'='%s': %s" % - (self._get_label_name(label_id), value, e)) - - return { 'spec_id': spec_id, 'value': value } - def _import_shape(self, shape): _shape = shape._asdict() label_id = self._get_label_id(_shape.pop('label')) @@ -482,7 +503,328 @@ def match_frame_fuzzy(self, path): return v return None -class CvatTaskDataExtractor(datumaro.SourceExtractor): +class ProjectData(InstanceLabelData): + LabeledShape = NamedTuple('LabledShape', [('type', str), ('frame', int), ('label', str), ('points', List[float]), ('occluded', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('task_id', int)]) + LabeledShape.__new__.__defaults__ = (0,0) + TrackedShape = NamedTuple('TrackedShape', + [('type', str), ('frame', int), ('points', List[float]), ('occluded', bool), ('outside', bool), ('keyframe', bool), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('z_order', int), ('label', str), ('track_id', int)], + ) + TrackedShape.__new__.__defaults__ = ('manual', 0, 0, None, 0) + Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)]) + Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)]) + Tag.__new__.__defaults__ = (0, ) + Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])]) + + def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None): + self._annotation_irs = annotation_irs + self._db_project = db_project + self._db_tasks: OrderedDict[int, Task] = OrderedDict( + ((db_task.id, db_task) for db_task in db_project.tasks.order_by("subset","id").all()) + ) + self._subsets = set() + self._host = host + self._create_callback = create_callback + self._MAX_ANNO_SIZE = 30000 + self._frame_info: Dict[Tuple[int, int], Literal["path", "width", "height", "subset"]] = dict() + self._frame_mapping: Dict[Tuple[str, str], Tuple[str, str]] = dict() + self._frame_steps: Dict[int, int] = {task.id: task.data.get_frame_step() for task in self._db_tasks.values()} + + for task in self._db_tasks.values(): + self._subsets.add(task.subset) + self._subsets: List[str] = list(self._subsets) + + InstanceLabelData.__init__(self, db_project) + + self._init_task_frame_offsets() + self._init_frame_info() + self._init_meta() + + def abs_frame_id(self, task_id: int, relative_id: int) -> int: + task = self._db_tasks[task_id] + if relative_id not in range(0, task.data.size): + raise ValueError(f"Unknown internal frame id {relative_id}") + return relative_id * task.data.get_frame_step() + task.data.start_frame + self._task_frame_offsets[task_id] + + def rel_frame_id(self, task_id: int, absolute_id: int) -> int: + task = self._db_tasks[task_id] + d, m = divmod( + absolute_id - task.data.start_frame, task.data.get_frame_step()) + if m or d not in range(0, task.data.size): + raise ValueError(f"Unknown frame {absolute_id}") + return d + + def _init_task_frame_offsets(self): + self._task_frame_offsets: Dict[int, int] = dict() + s = 0 + subset = None + + for task in self._db_tasks.values(): + if subset != task.subset: + s = 0 + subset = task.subset + self._task_frame_offsets[task.id] = s + s += task.data.start_frame + task.data.get_frame_step() * task.data.size + + + def _init_frame_info(self): + self._frame_info = dict() + original_names = DefaultDict[Tuple[str, str], int](int) + for task in self._db_tasks.values(): + defaulted_subset = get_defaulted_subset(task.subset, self._subsets) + if hasattr(task.data, 'video'): + self._frame_info.update({(task.id, frame): { + "path": "frame_{:06d}".format(self.abs_frame_id(task.id, frame)), + "width": task.data.video.width, + "height": task.data.video.height, + "subset": defaulted_subset, + } for frame in range(task.data.size)}) + else: + self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): { + "path": mangle_image_name(db_image.path, defaulted_subset, original_names), + "width": db_image.width, + "height": db_image.height, + "subset": defaulted_subset + } for db_image in task.data.images.all()}) + + self._frame_mapping = { + (self._db_tasks[frame_ident[0]].subset, self._get_filename(info["path"])): frame_ident + for frame_ident, info in self._frame_info.items() + } + + def _init_meta(self): + self._meta = OrderedDict([ + ('project', OrderedDict([ + ('id', str(self._db_project.id)), + ('name', self._db_project.name), + ("bugtracker", self._db_project.bug_tracker), + ("created", str(timezone.localtime(self._db_project.created_date))), + ("updated", str(timezone.localtime(self._db_project.updated_date))), + ("tasks", [ + ('task', + TaskData.meta_for_task(db_task, self._host) + ) for db_task in self._db_tasks.values() + ]), + + ("labels", [ + ("label", OrderedDict([ + ("name", db_label.name), + ("color", db_label.color), + ("attributes", [ + ("attribute", OrderedDict([ + ("name", db_attr.name), + ("mutable", str(db_attr.mutable)), + ("input_type", db_attr.input_type), + ("default_value", db_attr.default_value), + ("values", db_attr.values)])) + for db_attr in db_label.attributespec_set.all()]) + ])) for db_label in self._label_mapping.values() + ]), + + ("owner", OrderedDict([ + ("username", self._db_project.owner.username), + ("email", self._db_project.owner.email), + ]) if self._db_project.owner else ""), + + ("assignee", OrderedDict([ + ("username", self._db_project.assignee.username), + ("email", self._db_project.assignee.email), + ]) if self._db_project.assignee else ""), + ])), + ("dumped", str(timezone.localtime(timezone.now()))) + ]) + + def _export_tracked_shape(self, shape: dict, task_id: int): + return ProjectData.TrackedShape( + type=shape["type"], + frame=self.abs_frame_id(task_id, shape["frame"]), + label=self._get_label_name(shape["label_id"]), + points=shape["points"], + occluded=shape["occluded"], + z_order=shape.get("z_order", 0), + group=shape.get("group", 0), + outside=shape.get("outside", False), + keyframe=shape.get("keyframe", True), + track_id=shape["track_id"], + source=shape.get("source", "manual"), + attributes=self._export_attributes(shape["attributes"]), + ) + + def _export_labeled_shape(self, shape: dict, task_id: int): + return ProjectData.LabeledShape( + type=shape["type"], + label=self._get_label_name(shape["label_id"]), + frame=self.abs_frame_id(task_id, shape["frame"]), + points=shape["points"], + occluded=shape["occluded"], + z_order=shape.get("z_order", 0), + group=shape.get("group", 0), + source=shape["source"], + attributes=self._export_attributes(shape["attributes"]), + task_id=task_id, + ) + + def _export_tag(self, tag: dict, task_id: int): + return ProjectData.Tag( + frame=self.abs_frame_id(task_id, tag["frame"]), + label=self._get_label_name(tag["label_id"]), + group=tag.get("group", 0), + source=tag["source"], + attributes=self._export_attributes(tag["attributes"]), + task_id=task_id + ) + + def group_by_frame(self, include_empty=False): + frames: Dict[Tuple[str, int], ProjectData.Frame] = {} + def get_frame(task_id: int, idx: int) -> ProjectData.Frame: + frame_info = self._frame_info[(task_id, idx)] + abs_frame = self.abs_frame_id(task_id, idx) + if (frame_info["subset"], abs_frame) not in frames: + frames[(frame_info["subset"], abs_frame)] = ProjectData.Frame( + task_id=task_id, + subset=frame_info["subset"], + idx=idx, + frame=abs_frame, + name=frame_info["path"], + height=frame_info["height"], + width=frame_info["width"], + labeled_shapes=[], + tags=[], + ) + return frames[(frame_info["subset"], abs_frame)] + + if include_empty: + for ident in self._frame_info: + get_frame(*ident) + + for task in self._db_tasks.values(): + anno_manager = AnnotationManager(self._annotation_irs[task.id]) + for shape in sorted(anno_manager.to_shapes(task.data.size), + key=lambda shape: shape.get("z_order", 0)): + if (task.id, shape['frame']) not in self._frame_info: + continue + if 'track_id' in shape: + if shape['outside']: + continue + exported_shape = self._export_tracked_shape(shape, task.id) + else: + exported_shape = self._export_labeled_shape(shape, task.id) + get_frame(task.id, shape['frame']).labeled_shapes.append(exported_shape) + + for tag in self._annotation_irs[task.id].tags: + get_frame(task.id, tag['frame']).tags.append(self._export_tag(tag, task.id)) + + return iter(frames.values()) + + @property + def shapes(self): + for task in self._db_tasks.values(): + for shape in self._annotation_irs[task.id].shapes: + yield self._export_labeled_shape(shape, task.id) + + @property + def tracks(self): + idx = 0 + for task in self._db_tasks.values(): + for track in self._annotation_irs[task.id].tracks: + tracked_shapes = TrackManager.get_interpolated_shapes( + track, 0, task.data.size + ) + for tracked_shape in tracked_shapes: + tracked_shape["attributes"] += track["attributes"] + tracked_shape["track_id"] = idx + tracked_shape["group"] = track["group"] + tracked_shape["source"] = track["source"] + tracked_shape["label_id"] = track["label_id"] + yield ProjectData.Track( + label=self._get_label_name(track["label_id"]), + group=track["group"], + source=track["source"], + shapes=[self._export_tracked_shape(shape, task.id) + for shape in tracked_shapes], + task_id=task.id + ) + idx+=1 + + @property + def tags(self): + for task in self._db_tasks.values(): + for tag in self._annotation_irs[task.id].tags: + yield self._export_tag(tag, task.id) + + @property + def meta(self): + return self._meta + + @property + def data(self): + raise NotImplementedError() + + @property + def frame_info(self): + return self._frame_info + + @property + def frame_step(self): + return self._frame_steps + + @property + def db_project(self): + return self._db_project + + @property + def subsets(self) -> List[str]: + return self._subsets + + @property + def tasks(self): + return list(self._db_tasks.values()) + + @property + def task_data(self): + for task_id, task in self._db_tasks.items(): + yield TaskData(self._annotation_irs[task_id], task, self._host) + + @staticmethod + def _get_filename(path): + return osp.splitext(path)[0] + + +class CVATDataExtractorMixin: + def __init__(self): + super().__init__() + + def categories(self) -> dict: + raise NotImplementedError() + + @staticmethod + def _load_categories(labels: list): + categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {} + + label_categories = datumaro.LabelCategories(attributes=['occluded']) + + for _, label in labels: + label_categories.add(label['name']) + for _, attr in label['attributes']: + label_categories.attributes.add(attr['name']) + + categories[datumaro.AnnotationType.label] = label_categories + + return categories + + + def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list): + categories = self.categories() + label_cat = categories[datumaro.AnnotationType.label] + def map_label(name): return label_cat.find(name)[0] + label_attrs = { + label['name']: label['attributes'] + for _, label in labels + } + + return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label) + + +class CvatTaskDataExtractor(datumaro.SourceExtractor, CVATDataExtractorMixin): def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D): super().__init__() self._categories, self._user = self._load_categories(task_data, dimension=dimension) @@ -537,12 +879,13 @@ def _make_image(i, **kwargs): dm_image = _make_image(frame_data.idx, **image_args) else: dm_image = Image(**image_args) - dm_anno = self._read_cvat_anno(frame_data, task_data) + dm_anno = self._read_cvat_anno(frame_data, task_data.meta['task']['labels']) if dimension == DimensionType.DIM_2D: dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], - annotations=dm_anno, image=dm_image, - attributes={'frame': frame_data.frame}) + annotations=dm_anno, image=dm_image, + attributes={'frame': frame_data.frame + }) elif dimension == DimensionType.DIM_3D: attributes = {'frame': frame_data.frame} if format_type == "sly_pointcloud": @@ -564,18 +907,8 @@ def _make_image(i, **kwargs): self._items = dm_items - def __iter__(self): - for item in self._items: - yield item - - def __len__(self): - return len(self._items) - - def categories(self): - return self._categories - @staticmethod - def _load_categories(cvat_anno, dimension): + def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ categories = {} label_categories = datumaro.LabelCategories(attributes=['occluded']) @@ -595,102 +928,209 @@ def _load_categories(cvat_anno, dimension): return categories, user_info - def _read_cvat_anno(self, cvat_frame_anno, task_data): - item_anno = [] - + def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list): categories = self.categories() label_cat = categories[datumaro.AnnotationType.label] def map_label(name): return label_cat.find(name)[0] label_attrs = { label['name']: label['attributes'] - for _, label in task_data.meta['task']['labels'] + for _, label in labels } - def convert_attrs(label, cvat_attrs): - cvat_attrs = {a.name: a.value for a in cvat_attrs} - dm_attr = dict() - for _, a_desc in label_attrs[label]: - a_name = a_desc['name'] - a_value = cvat_attrs.get(a_name, a_desc['default_value']) - try: - if a_desc['input_type'] == AttributeType.NUMBER: - a_value = float(a_value) - elif a_desc['input_type'] == AttributeType.CHECKBOX: - a_value = (a_value.lower() == 'true') - dm_attr[a_name] = a_value - except Exception as e: - raise Exception( - "Failed to convert attribute '%s'='%s': %s" % - (a_name, a_value, e)) - if self._format_type == "sly_pointcloud" and (a_desc.get('input_type') == 'select' or a_desc.get('input_type') == 'radio'): - dm_attr[f"{a_name}__values"] = a_desc["values"] - - return dm_attr - - for tag_obj in cvat_frame_anno.tags: - anno_group = tag_obj.group or 0 - anno_label = map_label(tag_obj.label) - anno_attr = convert_attrs(tag_obj.label, tag_obj.attributes) - - anno = datumaro.Label(label=anno_label, - attributes=anno_attr, group=anno_group) - item_anno.append(anno) - - shapes = [] - for shape in cvat_frame_anno.shapes: - shapes.append({"id": shape.id, "label_id": shape.label_id}) + return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension) - for index, shape_obj in enumerate(cvat_frame_anno.labeled_shapes): - anno_group = shape_obj.group or 0 - anno_label = map_label(shape_obj.label) - anno_attr = convert_attrs(shape_obj.label, shape_obj.attributes) - anno_attr['occluded'] = shape_obj.occluded - - if hasattr(shape_obj, 'track_id'): - anno_attr['track_id'] = shape_obj.track_id - anno_attr['keyframe'] = shape_obj.keyframe - - anno_points = shape_obj.points - if shape_obj.type == ShapeType.POINTS: - anno = datumaro.Points(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.POLYLINE: - anno = datumaro.PolyLine(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.POLYGON: - anno = datumaro.Polygon(anno_points, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.RECTANGLE: - x0, y0, x1, y1 = anno_points - anno = datumaro.Bbox(x0, y0, x1 - x0, y1 - y0, - label=anno_label, attributes=anno_attr, group=anno_group, - z_order=shape_obj.z_order) - elif shape_obj.type == ShapeType.CUBOID: - if self._dimension == DimensionType.DIM_3D: - if self._format_type == "sly_pointcloud": - anno_id = shapes[index]["id"] - anno_attr["label_id"] = shapes[index]["label_id"] - else: - anno_id = index - position, rotation, scale = anno_points[0:3], anno_points[3:6], anno_points[6:9] - anno = datumaro.Cuboid3d(id=anno_id, position=position, rotation=rotation, scale=scale, - label=anno_label, attributes=anno_attr, group=anno_group - ) +class CVATProjectDataExtractor(datumaro.Extractor, CVATDataExtractorMixin): + def __init__(self, project_data: ProjectData, include_images: bool = False): + super().__init__() + self._categories = self._load_categories(project_data.meta['project']['labels']) + + dm_items: List[datumaro.DatasetItem] = [] + + ext_per_task: Dict[int, str] = {} + image_maker_per_task: Dict[int, Callable] = {} + + for task in project_data.tasks: + is_video = task.mode == 'interpolation' + ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else '' + if include_images: + frame_provider = FrameProvider(task.data) + if is_video: + # optimization for videos: use numpy arrays instead of bytes + # some formats or transforms can require image data + def image_maker_factory(frame_provider): + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.NUMPY_ARRAY)[0] + return Image(loader=loader, **kwargs) + return _make_image else: - continue + # for images use encoded data to avoid recoding + def image_maker_factory(frame_provider): + def _make_image(i, **kwargs): + loader = lambda _: frame_provider.get_frame(i, + quality=frame_provider.Quality.ORIGINAL, + out_type=frame_provider.Type.BUFFER)[0].getvalue() + return ByteImage(data=loader, **kwargs) + return _make_image + image_maker_per_task[task.id] = image_maker_factory(frame_provider) + + for frame_data in project_data.group_by_frame(include_empty=True): + image_args = { + 'path': frame_data.name + ext_per_task[frame_data.task_id], + 'size': (frame_data.height, frame_data.width), + } + if include_images: + dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args) else: - raise Exception("Unknown shape type '%s'" % shape_obj.type) + dm_image = Image(**image_args) + dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels']) + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotations=dm_anno, image=dm_image, + subset=frame_data.subset, + attributes={'frame': frame_data.frame} + ) + dm_items.append(dm_item) - item_anno.append(anno) + self._items = dm_items - return item_anno + def categories(self): + return self._categories + + def __iter__(self): + yield from self._items + + def __len__(self): + return len(self._items) + + +def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False): + if isinstance(instance_data, ProjectData): + return CVATProjectDataExtractor(instance_data, include_images) + else: + return CvatTaskDataExtractor(instance_data, include_images) class CvatImportError(Exception): pass +def mangle_image_name(name: str, subset: str, names: DefaultDict[Tuple[str, str], int]) -> str: + name, ext = name.rsplit(osp.extsep, maxsplit=1) + + if not names[(subset, name)]: + names[(subset, name)] += 1 + return osp.extsep.join([name, ext]) + else: + image_name = f"{name}_{names[(subset, name)]}" + if not names[(subset, image_name)]: + names[(subset, name)] += 1 + return osp.extsep.join([image_name, ext]) + else: + i = 1 + while i < sys.maxsize: + new_image_name = f"{image_name}_{i}" + if not names[(subset, new_image_name)]: + names[(subset, name)] += 1 + return osp.extsep.join([new_image_name, ext]) + i += 1 + raise Exception('Cannot mangle image name') + +def get_defaulted_subset(subset: str, subsets: List[str]) -> str: + if subset: + return subset + else: + if datumaro.DEFAULT_SUBSET_NAME not in subsets: + return datumaro.DEFAULT_SUBSET_NAME + else: + i = 1 + while i < sys.maxsize: + if f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' not in subsets: + return f'{datumaro.DEFAULT_SUBSET_NAME}_{i}' + i += 1 + raise Exception('Cannot find default name for subset') + + +def convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, format_name=None, dimension=DimensionType.DIM_2D): + item_anno = [] + + def convert_attrs(label, cvat_attrs): + cvat_attrs = {a.name: a.value for a in cvat_attrs} + dm_attr = dict() + for _, a_desc in label_attrs[label]: + a_name = a_desc['name'] + a_value = cvat_attrs.get(a_name, a_desc['default_value']) + try: + if a_desc['input_type'] == AttributeType.NUMBER: + a_value = float(a_value) + elif a_desc['input_type'] == AttributeType.CHECKBOX: + a_value = (a_value.lower() == 'true') + dm_attr[a_name] = a_value + except Exception as e: + raise Exception( + "Failed to convert attribute '%s'='%s': %s" % + (a_name, a_value, e)) + return dm_attr + + for tag_obj in cvat_frame_anno.tags: + anno_group = tag_obj.group or 0 + anno_label = map_label(tag_obj.label) + anno_attr = convert_attrs(tag_obj.label, tag_obj.attributes) + + anno = datumaro.Label(label=anno_label, + attributes=anno_attr, group=anno_group) + item_anno.append(anno) + + shapes = [] + if hasattr(cvat_frame_anno, 'shapes'): + for shape in cvat_frame_anno.shapes: + shapes.append({"id": shape.id, "label_id": shape.label_id}) + + for index, shape_obj in enumerate(cvat_frame_anno.labeled_shapes): + anno_group = shape_obj.group or 0 + anno_label = map_label(shape_obj.label) + anno_attr = convert_attrs(shape_obj.label, shape_obj.attributes) + anno_attr['occluded'] = shape_obj.occluded + + if hasattr(shape_obj, 'track_id'): + anno_attr['track_id'] = shape_obj.track_id + anno_attr['keyframe'] = shape_obj.keyframe + + anno_points = shape_obj.points + if shape_obj.type == ShapeType.POINTS: + anno = datumaro.Points(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.POLYLINE: + anno = datumaro.PolyLine(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.POLYGON: + anno = datumaro.Polygon(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.RECTANGLE: + x0, y0, x1, y1 = anno_points + anno = datumaro.Bbox(x0, y0, x1 - x0, y1 - y0, + label=anno_label, attributes=anno_attr, group=anno_group, + z_order=shape_obj.z_order) + elif shape_obj.type == ShapeType.CUBOID: + if dimension == DimensionType.DIM_3D: + if format_name == "sly_pointcloud": + anno_id = shapes[index]["id"] + else: + anno_id = index + position, rotation, scale = anno_points[0:3], anno_points[3:6], anno_points[6:9] + anno = datumaro.Cuboid3d(id=anno_id, position=position, rotation=rotation, scale=scale, + label=anno_label, attributes=anno_attr, group=anno_group + ) + else: + continue + else: + raise Exception("Unknown shape type '%s'" % shape_obj.type) + + item_anno.append(anno) + + return item_anno + def match_dm_item(item, task_data, root_hint=None): is_video = task_data.meta['task']['mode'] == 'interpolation' diff --git a/cvat/apps/dataset_manager/formats/camvid.py b/cvat/apps/dataset_manager/formats/camvid.py index a8fb50592e9a..2522f3fb226e 100644 --- a/cvat/apps/dataset_manager/formats/camvid.py +++ b/cvat/apps/dataset_manager/formats/camvid.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,13 +16,13 @@ @exporter(name='CamVid', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') dataset.transform('merge_instance_segments') - label_map = make_colormap(task_data) + label_map = make_colormap(instance_data) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'camvid', save_images=save_images, apply_colormap=True, @@ -31,10 +31,10 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='CamVid', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'camvid', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/coco.py b/cvat/apps/dataset_manager/formats/coco.py index 3e4fb223fdb4..927df2de567a 100644 --- a/cvat/apps/dataset_manager/formats/coco.py +++ b/cvat/apps/dataset_manager/formats/coco.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,9 +15,9 @@ @exporter(name='COCO', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'coco_instances', save_images=save_images, merge_images=True) @@ -25,14 +25,14 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='COCO', ext='JSON, ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): if zipfile.is_zipfile(src_file): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'coco', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) else: dataset = Dataset.import_from(src_file.name, 'coco_instances', env=dm_env) - import_dm_annotations(dataset, task_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index 786a5025e7c0..9e7fa514fb64 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -2,8 +2,10 @@ # # SPDX-License-Identifier: MIT +from io import BufferedWriter import os import os.path as osp +from typing import Callable import zipfile from collections import OrderedDict from glob import glob @@ -11,7 +13,7 @@ from datumaro.components.extractor import DatasetItem -from cvat.apps.dataset_manager.bindings import match_dm_item +from cvat.apps.dataset_manager.bindings import TaskData, match_dm_item, ProjectData, get_defaulted_subset from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.frame_provider import FrameProvider @@ -42,8 +44,10 @@ def _add_version(self): self.xmlgen.characters(self.version) self.xmlgen.endElement("version") - def open_root(self): + def open_document(self): self.xmlgen.startDocument() + + def open_root(self): self.xmlgen.startElement("annotations", {}) self._level += 1 self._add_version() @@ -168,23 +172,34 @@ def close_root(self): self._level -= 1 self._indent() self.xmlgen.endElement("annotations") + self._indent() + + def close_document(self): self.xmlgen.endDocument() + return XmlAnnotationWriter(file_object) -def dump_as_cvat_annotation(file_object, annotations): - dumper = create_xml_dumper(file_object) +def dump_as_cvat_annotation(dumper, annotations): dumper.open_root() dumper.add_meta(annotations.meta) for frame_annotation in annotations.group_by_frame(include_empty=True): frame_id = frame_annotation.frame - dumper.open_image(OrderedDict([ + image_attrs = OrderedDict([ ("id", str(frame_id)), ("name", frame_annotation.name), + ]) + if isinstance(annotations, ProjectData): + image_attrs.update(OrderedDict([ + ("subset", frame_annotation.subset), + ("task_id", str(frame_annotation.task_id)), + ])) + image_attrs.update(OrderedDict([ ("width", str(frame_annotation.width)), ("height", str(frame_annotation.height)) ])) + dumper.open_image(image_attrs) for shape in frame_annotation.labeled_shapes: dump_data = OrderedDict([ @@ -286,8 +301,7 @@ def dump_as_cvat_annotation(file_object, annotations): dumper.close_image() dumper.close_root() -def dump_as_cvat_interpolation(file_object, annotations): - dumper = create_xml_dumper(file_object) +def dump_as_cvat_interpolation(dumper, annotations): dumper.open_root() dumper.add_meta(annotations.meta) def dump_track(idx, track): @@ -298,6 +312,13 @@ def dump_track(idx, track): ("source", track.source), ]) + if hasattr(track, 'task_id'): + task, = filter(lambda task: task.id == track.task_id, annotations.tasks) + dump_data.update(OrderedDict([ + ('task_id', str(track.task_id)), + ('subset', get_defaulted_subset(task.subset, annotations.subsets)), + ])) + if track.group: dump_data['group_id'] = str(track.group) dumper.open_track(dump_data) @@ -383,11 +404,17 @@ def dump_track(idx, track): counter += 1 for shape in annotations.shapes: - dump_track(counter, annotations.Track( - label=shape.label, - group=shape.group, - source=shape.source, - shapes=[annotations.TrackedShape( + frame_step = annotations.frame_step if isinstance(annotations, TaskData) else annotations.frame_step[shape.task_id] + if isinstance(annotations, TaskData): + stop_frame = int(annotations.meta['task']['stop_frame']) + else: + task_meta = list(filter(lambda task: int(task[1]['id']) == shape.task_id, annotations.meta['project']['tasks']))[0][1] + stop_frame = int(task_meta['stop_frame']) + track = { + 'label': shape.label, + 'group': shape.group, + 'source': shape.source, + 'shapes': [annotations.TrackedShape( type=shape.type, points=shape.points, occluded=shape.occluded, @@ -405,13 +432,15 @@ def dump_track(idx, track): outside=True, keyframe=True, z_order=shape.z_order, - frame=shape.frame + annotations.frame_step, + frame=shape.frame + frame_step, attributes=shape.attributes, - )] if shape.frame + annotations.frame_step < \ - int(annotations.meta['task']['stop_frame']) \ + )] if shape.frame + frame_step < \ + stop_frame \ else [] ), - )) + } + if isinstance(annotations, ProjectData): track['task_id'] = shape.task_id + dump_track(counter, annotations.Track(**track)) counter += 1 dumper.close_root() @@ -527,39 +556,76 @@ def load(file_object, annotations): tag = None el.clear() -def _export(dst_file, task_data, anno_callback, save_images=False): +def dump_task_anno(dst_file, task_data, callback): + dumper = create_xml_dumper(dst_file) + dumper.open_document() + callback(dumper, task_data) + dumper.close_document() + +def dump_project_anno(dst_file: BufferedWriter, project_data: ProjectData, callback: Callable): + dumper = create_xml_dumper(dst_file) + dumper.open_document() + callback(dumper, project_data) + dumper.close_document() + +def dump_media_files(task_data: TaskData, img_dir: str, project_data: ProjectData = None): + ext = '' + if task_data.meta['task']['mode'] == 'interpolation': + ext = FrameProvider.VIDEO_FRAME_EXT + + frame_provider = FrameProvider(task_data.db_task.data) + frames = frame_provider.get_frames( + frame_provider.Quality.ORIGINAL, + frame_provider.Type.BUFFER) + for frame_id, (frame_data, _) in enumerate(frames): + frame_name = task_data.frame_info[frame_id]['path'] if project_data is None \ + else project_data.frame_info[(task_data.db_task.id, frame_id)]['path'] + img_path = osp.join(img_dir, frame_name + ext) + os.makedirs(osp.dirname(img_path), exist_ok=True) + with open(img_path, 'wb') as f: + f.write(frame_data.getvalue()) + +def _export_task(dst_file, task_data, anno_callback, save_images=False): with TemporaryDirectory() as temp_dir: with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f: - anno_callback(f, task_data) + dump_task_anno(f, task_data, anno_callback) if save_images: - ext = '' - if task_data.meta['task']['mode'] == 'interpolation': - ext = FrameProvider.VIDEO_FRAME_EXT - - img_dir = osp.join(temp_dir, 'images') - frame_provider = FrameProvider(task_data.db_task.data) - frames = frame_provider.get_frames( - frame_provider.Quality.ORIGINAL, - frame_provider.Type.BUFFER) - for frame_id, (frame_data, _) in enumerate(frames): - frame_name = task_data.frame_info[frame_id]['path'] - img_path = osp.join(img_dir, frame_name + ext) - os.makedirs(osp.dirname(img_path), exist_ok=True) - with open(img_path, 'wb') as f: - f.write(frame_data.getvalue()) + dump_media_files(task_data, osp.join(temp_dir, 'images')) + + make_zip_archive(temp_dir, dst_file) + +def _export_project(dst_file: str, project_data: ProjectData, anno_callback: Callable, save_images: bool=False): + with TemporaryDirectory() as temp_dir: + with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f: + dump_project_anno(f, project_data, anno_callback) + + if save_images: + for task_data in project_data.task_data: + subset = get_defaulted_subset(task_data.db_task.subset, project_data.subsets) + subset_dir = osp.join(temp_dir, 'images', subset) + os.makedirs(subset_dir, exist_ok=True) + dump_media_files(task_data, subset_dir, project_data) make_zip_archive(temp_dir, dst_file) @exporter(name='CVAT for video', ext='ZIP', version='1.1') -def _export_video(dst_file, task_data, save_images=False): - _export(dst_file, task_data, - anno_callback=dump_as_cvat_interpolation, save_images=save_images) +def _export_video(dst_file, instance_data, save_images=False): + if isinstance(instance_data, ProjectData): + _export_project(dst_file, instance_data, + anno_callback=dump_as_cvat_interpolation, save_images=save_images) + else: + _export_task(dst_file, instance_data, + anno_callback=dump_as_cvat_interpolation, save_images=save_images) @exporter(name='CVAT for images', ext='ZIP', version='1.1') -def _export_images(dst_file, task_data, save_images=False): - _export(dst_file, task_data, - anno_callback=dump_as_cvat_annotation, save_images=save_images) +def _export_images(dst_file, instance_data, save_images=False): + if isinstance(instance_data, ProjectData): + _export_project(dst_file, instance_data, + anno_callback=dump_as_cvat_annotation, save_images=save_images) + else: + _export_task(dst_file, instance_data, + anno_callback=dump_as_cvat_annotation, save_images=save_images) @importer(name='CVAT', ext='XML, ZIP', version='1.1') def _import(src_file, task_data): diff --git a/cvat/apps/dataset_manager/formats/datumaro/__init__.py b/cvat/apps/dataset_manager/formats/datumaro/__init__.py index 0f351f83b608..f4fe0423345b 100644 --- a/cvat/apps/dataset_manager/formats/datumaro/__init__.py +++ b/cvat/apps/dataset_manager/formats/datumaro/__init__.py @@ -8,8 +8,8 @@ import shutil from tempfile import TemporaryDirectory -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, + import_dm_annotations, ProjectData) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.settings.base import BASE_DIR from datumaro.components.project import Project @@ -23,23 +23,28 @@ class DatumaroProjectExporter: _TEMPLATES_DIR = osp.join(osp.dirname(__file__), 'export_templates') @staticmethod - def _save_image_info(save_dir, task_data): + def _save_image_info(save_dir, instance_data): os.makedirs(save_dir, exist_ok=True) config = { - 'server_url': task_data._host or 'localhost', - 'task_id': task_data.db_task.id, + 'server_url': instance_data._host or 'localhost' } + if isinstance(instance_data, ProjectData): + config['project_id'] = instance_data.db_project.id + else: + config['task_id'] = instance_data.db_task.id images = [] images_meta = { 'images': images, } - for frame_id, frame in task_data.frame_info.items(): - images.append({ + for frame_id, frame in enumerate(instance_data.frame_info.values()): + image_info = { 'id': frame_id, 'name': osp.basename(frame['path']), 'width': frame['width'], 'height': frame['height'], - }) + } + if isinstance(instance_data, ProjectData): + image_info['subset'] = frame['subset'] with open(osp.join(save_dir, 'config.json'), 'w', encoding='utf-8') as config_file: @@ -48,11 +53,12 @@ def _save_image_info(save_dir, task_data): 'w', encoding='utf-8') as images_file: json.dump(images_meta, images_file) - def _export(self, task_data, save_dir, save_images=False): - dataset = CvatTaskDataExtractor(task_data, include_images=save_images) + def _export(self, instance_data, save_dir, save_images=False): + dataset = GetCVATDataExtractor(instance_data, include_images=save_images) + db_instance = instance_data.db_project if isinstance(instance_data, ProjectData) else instance_data.db_task dm_env.converters.get('datumaro_project').convert(dataset, save_dir=save_dir, save_images=save_images, - project_config={ 'project_name': task_data.db_task.name, } + project_config={ 'project_name': db_instance.name, } ) project = Project.load(save_dir) @@ -64,13 +70,16 @@ def _export(self, task_data, save_dir, save_images=False): if not save_images: # add remote links to images - source_name = 'task_%s_images' % task_data.db_task.id + source_name = '{}_{}_images'.format( + 'project' if isinstance(instance_data, ProjectData) else 'task', + db_instance.id, + ) project.add_source(source_name, { 'format': self._REMOTE_IMAGES_EXTRACTOR, }) self._save_image_info( osp.join(save_dir, project.local_source_dir(source_name)), - task_data) + instance_data) project.save() templates_dir = osp.join(self._TEMPLATES_DIR, 'plugins') @@ -87,7 +96,7 @@ def _export(self, task_data, save_dir, save_images=False): shutil.copytree(osp.join(BASE_DIR, 'utils', 'cli'), osp.join(cvat_utils_dst_dir, 'cli')) - def __call__(self, dst_file, task_data, save_images=False): + def __call__(self, dst_file, instance_data, save_images=False): with TemporaryDirectory() as temp_dir: - self._export(task_data, save_dir=temp_dir, save_images=save_images) + self._export(instance_data, save_dir=temp_dir, save_images=save_images) make_zip_archive(temp_dir, dst_file) diff --git a/cvat/apps/dataset_manager/formats/icdar.py b/cvat/apps/dataset_manager/formats/icdar.py index 03eda245432a..544e20decf88 100644 --- a/cvat/apps/dataset_manager/formats/icdar.py +++ b/cvat/apps/dataset_manager/formats/icdar.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Caption, Label, LabelCategories, ItemTransform) -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -75,45 +75,45 @@ def transform_item(self, item): return item.wrap(annotations=annotations) @exporter(name='ICDAR Recognition', ext='ZIP', version='1.0') -def _export_recognition(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_recognition(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform(LabelToCaption) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'icdar_word_recognition', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Recognition', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_word_recognition', env=dm_env) dataset.transform(CaptionToLabel, 'icdar') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) @exporter(name='ICDAR Localization', ext='ZIP', version='1.0') -def _export_localization(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_localization(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'icdar_text_localization', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Localization', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_text_localization', env=dm_env) dataset.transform(AddLabelToAnns, 'icdar') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) @exporter(name='ICDAR Segmentation', ext='ZIP', version='1.0') -def _export_segmentation(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export_segmentation(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') @@ -122,10 +122,10 @@ def _export_segmentation(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='ICDAR Segmentation', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'icdar_text_segmentation', env=dm_env) dataset.transform(AddLabelToAnns, 'icdar') dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/imagenet.py b/cvat/apps/dataset_manager/formats/imagenet.py index 2ed0cb474849..1085ef745355 100644 --- a/cvat/apps/dataset_manager/formats/imagenet.py +++ b/cvat/apps/dataset_manager/formats/imagenet.py @@ -9,7 +9,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -17,9 +17,9 @@ @exporter(name='ImageNet', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: if save_images: dataset.export(temp_dir, 'imagenet', save_images=save_images) @@ -29,11 +29,11 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='ImageNet', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) if glob(osp.join(tmp_dir, '*.txt')): dataset = Dataset.import_from(tmp_dir, 'imagenet_txt', env=dm_env) else: dataset = Dataset.import_from(tmp_dir, 'imagenet', env=dm_env) - import_dm_annotations(dataset, task_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/labelme.py b/cvat/apps/dataset_manager/formats/labelme.py index 744b11faab0b..2fc1f7f73c4b 100644 --- a/cvat/apps/dataset_manager/formats/labelme.py +++ b/cvat/apps/dataset_manager/formats/labelme.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,19 +15,19 @@ @exporter(name='LabelMe', ext='ZIP', version='3.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'label_me', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='LabelMe', ext='ZIP', version='3.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'label_me', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/market1501.py b/cvat/apps/dataset_manager/formats/market1501.py index f94d65dca88a..f578d3ab10e1 100644 --- a/cvat/apps/dataset_manager/formats/market1501.py +++ b/cvat/apps/dataset_manager/formats/market1501.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import (AnnotationType, Label, LabelCategories, ItemTransform) -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -60,19 +60,19 @@ def transform_item(self, item): @exporter(name='Market-1501', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.transform(LabelAttrToAttr, 'market-1501') dataset.export(temp_dir, 'market1501', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='Market-1501', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'market1501', env=dm_env) dataset.transform(AttrToLabelAttr, 'market-1501') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index 3e3780e8c6a7..67d61eed3591 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive @@ -16,23 +16,23 @@ @exporter(name='Segmentation mask', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') dataset.transform('merge_instance_segments') with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'voc_segmentation', save_images=save_images, - apply_colormap=True, label_map=make_colormap(task_data)) + apply_colormap=True, label_map=make_colormap(instance_data)) make_zip_archive(temp_dir, dst_file) @importer(name='Segmentation mask', ext='ZIP', version='1.1') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/mot.py b/cvat/apps/dataset_manager/formats/mot.py index 29d5182a674e..26fc7b0db616 100644 --- a/cvat/apps/dataset_manager/formats/mot.py +++ b/cvat/apps/dataset_manager/formats/mot.py @@ -8,16 +8,16 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer @exporter(name='MOT', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'mot_seq_gt', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/mots.py b/cvat/apps/dataset_manager/formats/mots.py index b8b562ec900a..fc2d69edea99 100644 --- a/cvat/apps/dataset_manager/formats/mots.py +++ b/cvat/apps/dataset_manager/formats/mots.py @@ -8,7 +8,7 @@ from datumaro.components.extractor import AnnotationType, ItemTransform from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, find_dataset_root, match_dm_item) from cvat.apps.dataset_manager.util import make_zip_archive @@ -21,9 +21,9 @@ def transform_item(self, item): if 'track_id' in a.attributes]) @exporter(name='MOTS PNG', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) dataset.transform(KeepTracks) # can only export tracks dataset.transform('polygons_to_masks') dataset.transform('boxes_to_masks') diff --git a/cvat/apps/dataset_manager/formats/pascal_voc.py b/cvat/apps/dataset_manager/formats/pascal_voc.py index 3f10b93aa937..93504628eb37 100644 --- a/cvat/apps/dataset_manager/formats/pascal_voc.py +++ b/cvat/apps/dataset_manager/formats/pascal_voc.py @@ -11,17 +11,17 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, + ProjectData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer @exporter(name='PASCAL VOC', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'voc', save_images=save_images, label_map='source') @@ -29,15 +29,16 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) @importer(name='PASCAL VOC', ext='ZIP', version='1.1') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) # put label map from the task if not present labelmap_file = osp.join(tmp_dir, 'labelmap.txt') if not osp.isfile(labelmap_file): - labels = (label['name'] + ':::' - for _, label in task_data.meta['task']['labels']) + labels_meta = instance_data.meta['project']['labels'] \ + if isinstance(instance_data, ProjectData) else instance_data.meta['task']['labels'] + labels = (label['name'] + ':::' for _, label in labels_meta) with open(labelmap_file, 'w') as f: f.write('\n'.join(labels)) @@ -57,4 +58,4 @@ def _import(src_file, task_data): dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 1fc31e4a6efe..458029a132d2 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType @@ -18,6 +18,9 @@ @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily") + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) diff --git a/cvat/apps/dataset_manager/formats/tfrecord.py b/cvat/apps/dataset_manager/formats/tfrecord.py index 9847bf61b66e..d9c705a75422 100644 --- a/cvat/apps/dataset_manager/formats/tfrecord.py +++ b/cvat/apps/dataset_manager/formats/tfrecord.py @@ -6,7 +6,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.project import Dataset @@ -23,18 +23,18 @@ @exporter(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'tf_detection_api', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'tf_detection_api', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/utils.py b/cvat/apps/dataset_manager/formats/utils.py index 184a133161b6..0d545e465831 100644 --- a/cvat/apps/dataset_manager/formats/utils.py +++ b/cvat/apps/dataset_manager/formats/utils.py @@ -48,8 +48,9 @@ def rgb2hex(color): def hex2rgb(color): return tuple(int(color.lstrip('#')[i:i+2], 16) for i in (0, 2, 4)) -def make_colormap(task_data): - labels = [label for _, label in task_data.meta['task']['labels']] +def make_colormap(instance_data): + instance_name = 'project' if 'project' in instance_data.meta.keys() else 'task' + labels = [label for _, label in instance_data.meta[instance_name]['labels']] label_names = [label['name'] for label in labels] if 'background' not in label_names: diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 12eafbce90cf..7384f7beabed 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData, \ import_dm_annotations from .registry import dm_env @@ -20,6 +20,9 @@ @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, task_data, save_images=False): + if not isinstance(task_data, TaskData): + raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily") + dataset = Dataset.from_extractors(CvatTaskDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) diff --git a/cvat/apps/dataset_manager/formats/vggface2.py b/cvat/apps/dataset_manager/formats/vggface2.py index 0ae6d9a9e531..d75f960afadb 100644 --- a/cvat/apps/dataset_manager/formats/vggface2.py +++ b/cvat/apps/dataset_manager/formats/vggface2.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,19 +15,19 @@ @exporter(name='VGGFace2', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'vgg_face2', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='VGGFace2', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'vgg_face2', env=dm_env) dataset.transform('rename', r"|([^/]+/)?(.+)|\2|") - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/widerface.py b/cvat/apps/dataset_manager/formats/widerface.py index 7f120ffe2154..b578c14c7cee 100644 --- a/cvat/apps/dataset_manager/formats/widerface.py +++ b/cvat/apps/dataset_manager/formats/widerface.py @@ -7,7 +7,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, \ +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive @@ -15,18 +15,18 @@ @exporter(name='WiderFace', ext='ZIP', version='1.0') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'wider_face', save_images=save_images) make_zip_archive(temp_dir, dst_file) @importer(name='WiderFace', ext='ZIP', version='1.0') -def _import(src_file, task_data): +def _import(src_file, instance_data): with TemporaryDirectory() as tmp_dir: zipfile.ZipFile(src_file).extractall(tmp_dir) dataset = Dataset.import_from(tmp_dir, 'wider_face', env=dm_env) - import_dm_annotations(dataset, task_data) + import_dm_annotations(dataset, instance_data) diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py index 0df6f5fe27a1..6327f3c04fb6 100644 --- a/cvat/apps/dataset_manager/formats/yolo.py +++ b/cvat/apps/dataset_manager/formats/yolo.py @@ -8,7 +8,7 @@ from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, import_dm_annotations, match_dm_item, find_dataset_root) from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.extractor import DatasetItem @@ -19,9 +19,9 @@ @exporter(name='YOLO', ext='ZIP', version='1.1') -def _export(dst_file, task_data, save_images=False): - dataset = Dataset.from_extractors(CvatTaskDataExtractor( - task_data, include_images=save_images), env=dm_env) +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) with TemporaryDirectory() as temp_dir: dataset.export(temp_dir, 'yolo', save_images=save_images) diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py new file mode 100644 index 000000000000..866a75d47e8b --- /dev/null +++ b/cvat/apps/dataset_manager/project.py @@ -0,0 +1,71 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from typing import Callable + +from django.db import transaction + +from cvat.apps.engine import models +from cvat.apps.dataset_manager.task import TaskAnnotation + +from .annotation import AnnotationIR +from .bindings import ProjectData +from .formats.registry import make_exporter + +def export_project(project_id, dst_file, format_name, + server_url=None, save_images=False): + # For big tasks dump function may run for a long time and + # we dont need to acquire lock after the task has been initialized from DB. + # But there is the bug with corrupted dump file in case 2 or + # more dump request received at the same time: + # https://github.com/opencv/cvat/issues/217 + with transaction.atomic(): + project = ProjectAnnotation(project_id) + project.init_from_db() + + exporter = make_exporter(format_name) + with open(dst_file, 'wb') as f: + project.export(f, exporter, host=server_url, save_images=save_images) + +class ProjectAnnotation: + def __init__(self, pk: int): + self.db_project = models.Project.objects.get(id=pk) + self.db_tasks = models.Task.objects.filter(project__id=pk).order_by('id') + + self.annotation_irs: dict[int, AnnotationIR] = dict() + + def reset(self): + for annotation_ir in self.annotation_irs.values(): + annotation_ir.reset() + + def put(self, data): + raise NotImplementedError() + + def create(self, data): + raise NotImplementedError() + + def update(self, data): + raise NotImplementedError() + + def delete(self, data=None): + raise NotImplementedError() + + def init_from_db(self): + self.reset() + + for task in self.db_tasks: + annotation = TaskAnnotation(pk=task.id) + annotation.init_from_db() + self.annotation_irs[task.id] = annotation.ir_data + + def export(self, dst_file: str, exporter: Callable, host: str='', **options): + project_data = ProjectData( + annotation_irs=self.annotation_irs, + db_project=self.db_project, + host=host + ) + exporter(dst_file, project_data, **options) + @property + def data(self) -> dict: + raise NotImplementedError() \ No newline at end of file diff --git a/cvat/apps/dataset_manager/tests/assets/projects.json b/cvat/apps/dataset_manager/tests/assets/projects.json new file mode 100644 index 000000000000..cef9a4ba891f --- /dev/null +++ b/cvat/apps/dataset_manager/tests/assets/projects.json @@ -0,0 +1,55 @@ +{ + "main": { + "name": "Main project", + "owner_id": 1, + "assignee_id": 2, + "labels": [ + { + "name": "car", + "color": "#2080c0", + "attributes": [ + { + "name": "select_name", + "mutable": false, + "input_type": "select", + "default_value": "bmw", + "values": ["bmw", "mazda", "renault"] + }, + { + "name": "radio_name", + "mutable": false, + "input_type": "radio", + "default_value": "x1", + "values": ["x1", "x2", "x3"] + }, + { + "name": "check_name", + "mutable": true, + "input_type": "checkbox", + "default_value": "false", + "values": ["false"] + }, + { + "name": "text_name", + "mutable": false, + "input_type": "text", + "default_value": "qwerty", + "values": ["qwerty"] + }, + { + "name": "number_name", + "mutable": false, + "input_type": "number", + "default_value": "-4", + "values": ["-4", "4", "1"] + } + ] + }, + { + "name": "person", + "color": "#c06060", + "attributes": [] + } + ] + } +} diff --git a/cvat/apps/dataset_manager/tests/assets/tasks.json b/cvat/apps/dataset_manager/tests/assets/tasks.json index 09e2d866287d..23ea55e2ae7f 100644 --- a/cvat/apps/dataset_manager/tests/assets/tasks.json +++ b/cvat/apps/dataset_manager/tests/assets/tasks.json @@ -282,17 +282,20 @@ } ] }, - "many jobs": { - "name": "many jobs", + "task in project #1": { + "name": "First task in project", + "project_id": 1, "overlap": 0, - "segment_size": 5, + "segment_size": 100, "owner_id": 1, - "labels": [ - { - "name": "car", - "color": "#2080c0", - "attributes": [] - } - ] + "assignee_id": 2 + }, + "task in project #2": { + "name": "Second task in project", + "project_id": 1, + "overlap": 0, + "segment_size": 100, + "owner_id": 1, + "assignee_id": 2 } } diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 2d69aee8c58d..313a392b0c26 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -7,6 +7,7 @@ import os.path as osp import os import av +from django.http import response import numpy as np import random import xml.etree.ElementTree as ET @@ -26,6 +27,10 @@ from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.engine.models import Task +projects_path = osp.join(osp.dirname(__file__), 'assets', 'projects.json') +with open(projects_path) as file: + projects = json.load(file) + tasks_path = osp.join(osp.dirname(__file__), 'assets', 'tasks.json') with open(tasks_path) as file: tasks = json.load(file) @@ -133,8 +138,8 @@ def _put_api_v1_job_id_annotations(self, jid, data): return response @staticmethod - def _generate_task_images(count): # pylint: disable=no-self-use - images = {"client_files[%d]" % i: generate_image_file("image_%d.jpg" % i) for i in range(count)} + def _generate_task_images(count, name_offsets = 0): # pylint: disable=no-self-use + images = {"client_files[%d]" % i: generate_image_file("image_%d.jpg" % (i + name_offsets)) for i in range(count)} images["image_quality"] = 75 return images @@ -159,6 +164,14 @@ def _create_task(self, data, image_data): return task + def _create_project(self, data): + with ForceLogin(self.user, self.client): + response = self.client.post('/api/v1/projects', data=data, format="json") + assert response.status_code == status.HTTP_201_CREATED, response.status_code + project = response.data + + return project + def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): response = self.client.get("/api/v1/tasks/{}/jobs".format(task_id)) @@ -297,14 +310,25 @@ def _generate_url_dump_job_annotations(self, job_id): def _generate_url_upload_job_annotations(self, job_id, upload_format_name): return f"/api/v1/jobs/{job_id}/annotations?format={upload_format_name}" - def _generate_url_dump_dataset(self, task_id): + def _generate_url_dump_task_dataset(self, task_id): return f"/api/v1/tasks/{task_id}/dataset" + def _generate_url_dump_project_annotations(self, project_id, format_name): + return f"/api/v1/projects/{project_id}/annotations?format={format_name}" + + def _generate_url_dump_project_dataset(self, project_id, format_name): + return f"/api/v1/projects/{project_id}/dataset?format={format_name}" + def _remove_annotations(self, url, user): response = self._delete_request(url, user) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) return response + def _delete_project(self, project_id, user): + response = self._delete_request(f'/api/v1/projects/{project_id}', user) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + return response + class TaskDumpUploadTest(_DbTestBase): def test_api_v1_dump_and_upload_annotations_with_objects_type_is_shape(self): @@ -789,7 +813,7 @@ def test_api_v1_export_dataset(self): task = self._create_task(tasks["main"], images) task_id = task["id"] # dump annotations - url = self._generate_url_dump_dataset(task_id) + url = self._generate_url_dump_task_dataset(task_id) for user, edata in list(expected.items()): user_name = edata['name'] file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') @@ -1147,3 +1171,108 @@ def test_api_v1_check_attribute_import_in_tracks(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + +class ProjectDump(_DbTestBase): + def test_api_v1_export_dataset(self): + test_name = self._testMethodName + dump_formats = dm.views.get_export_formats() + + expected = { + self.admin: {'name': 'admin', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + self.user: {'name': 'user', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + None: {'name': 'none', 'code': status.HTTP_401_UNAUTHORIZED, 'create code': status.HTTP_401_UNAUTHORIZED, + 'accept code': status.HTTP_401_UNAUTHORIZED, 'file_exists': False}, + } + + with TestDir() as test_dir: + for dump_format in dump_formats: + if not dump_format.ENABLED or dump_format.DIMENSION == dm.bindings.DimensionType.DIM_3D: + continue + dump_format_name = dump_format.DISPLAY_NAME + with self.subTest(format=dump_format_name): + project = self._create_project(projects['main']) + pid = project['id'] + images = self._generate_task_images(3) + tasks['task in project #1']['project_id'] = pid + self._create_task(tasks['task in project #1'], images) + images = self._generate_task_images(3, 3) + tasks['task in project #2']['project_id'] = pid + self._create_task(tasks['task in project #2'], images) + url = self._generate_url_dump_project_dataset(project['id'], dump_format_name) + + for user, edata in list(expected.items()): + user_name = edata['name'] + file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') + data = { + "format": dump_format_name, + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["accept code"]) + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["create code"]) + data = { + "format": dump_format_name, + "action": "download", + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["code"]) + if response.status_code == status.HTTP_200_OK: + content = BytesIO(b"".join(response.streaming_content)) + with open(file_zip_name, "wb") as f: + f.write(content.getvalue()) + self.assertEqual(response.status_code, edata['code']) + self.assertEqual(osp.exists(file_zip_name), edata['file_exists']) + + def test_api_v1_export_annotatios(self): + test_name = self._testMethodName + dump_formats = dm.views.get_export_formats() + + expected = { + self.admin: {'name': 'admin', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + self.user: {'name': 'user', 'code': status.HTTP_200_OK, 'create code': status.HTTP_201_CREATED, + 'accept code': status.HTTP_202_ACCEPTED, 'file_exists': True}, + None: {'name': 'none', 'code': status.HTTP_401_UNAUTHORIZED, 'create code': status.HTTP_401_UNAUTHORIZED, + 'accept code': status.HTTP_401_UNAUTHORIZED, 'file_exists': False}, + } + + with TestDir() as test_dir: + for dump_format in dump_formats: + if not dump_format.ENABLED or dump_format.DIMENSION == dm.bindings.DimensionType.DIM_3D: + continue + dump_format_name = dump_format.DISPLAY_NAME + with self.subTest(format=dump_format_name): + project = self._create_project(projects['main']) + pid = project['id'] + images = self._generate_task_images(3) + tasks['task in project #1']['project_id'] = pid + self._create_task(tasks['task in project #1'], images) + images = self._generate_task_images(3, 3) + tasks['task in project #2']['project_id'] = pid + self._create_task(tasks['task in project #2'], images) + url = self._generate_url_dump_project_annotations(project['id'], dump_format_name) + + for user, edata in list(expected.items()): + user_name = edata['name'] + file_zip_name = osp.join(test_dir, f'{test_name}_{user_name}_{dump_format_name}.zip') + data = { + "format": dump_format_name, + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["accept code"]) + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["create code"]) + data = { + "format": dump_format_name, + "action": "download", + } + response = self._get_request_with_data(url, data, user) + self.assertEqual(response.status_code, edata["code"]) + if response.status_code == status.HTTP_200_OK: + content = BytesIO(b"".join(response.streaming_content)) + with open(file_zip_name, "wb") as f: + f.write(content.getvalue()) + self.assertEqual(response.status_code, edata['code']) + self.assertEqual(osp.exists(file_zip_name), edata['file_exists']) diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index 36fcea63fd3a..4f51c69a91ef 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -13,9 +13,10 @@ from django.utils import timezone import cvat.apps.dataset_manager.task as task -from cvat.apps.engine.backup import TaskExporter +import cvat.apps.dataset_manager.project as project from cvat.apps.engine.log import slogger -from cvat.apps.engine.models import Task +from cvat.apps.engine.models import Project, Task +from cvat.apps.engine.backup import TaskExporter from .formats.registry import EXPORT_FORMATS, IMPORT_FORMATS from .util import current_function_name @@ -29,22 +30,32 @@ def log_exception(logger=None, exc_info=True): exc_info=exc_info) -def get_export_cache_dir(db_task): - task_dir = osp.abspath(db_task.get_task_dirname()) - if osp.isdir(task_dir): - return osp.join(task_dir, 'export_cache') +def get_export_cache_dir(db_instance): + base_dir = osp.abspath(db_instance.get_project_dirname() if isinstance(db_instance, Project) else db_instance.get_task_dirname()) + if osp.isdir(base_dir): + return osp.join(base_dir, 'export_cache') else: - raise Exception('Task dir {} does not exist'.format(task_dir)) + raise Exception('{} dir {} does not exist'.format("Project" if isinstance(db_instance, Project) else "Task", base_dir)) DEFAULT_CACHE_TTL = timedelta(hours=10) -CACHE_TTL = DEFAULT_CACHE_TTL +TASK_CACHE_TTL = DEFAULT_CACHE_TTL +PROJECT_CACHE_TTL = DEFAULT_CACHE_TTL / 3 -def export_task(task_id, dst_format, server_url=None, save_images=False): +def export(dst_format, task_id=None, project_id=None, server_url=None, save_images=False): try: - db_task = Task.objects.get(pk=task_id) - - cache_dir = get_export_cache_dir(db_task) + if task_id is not None: + db_instance = Task.objects.get(pk=task_id) + logger = slogger.task[task_id] + cache_ttl = TASK_CACHE_TTL + export_fn = task.export_task + else: + db_instance = Project.objects.get(pk=project_id) + logger = slogger.project[project_id] + cache_ttl = PROJECT_CACHE_TTL + export_fn = project.export_project + + cache_dir = get_export_cache_dir(db_instance) exporter = EXPORT_FORMATS[dst_format] output_base = '%s_%s' % ('dataset' if save_images else 'annotations', @@ -52,39 +63,51 @@ def export_task(task_id, dst_format, server_url=None, save_images=False): output_path = '%s.%s' % (output_base, exporter.EXT) output_path = osp.join(cache_dir, output_path) - task_time = timezone.localtime(db_task.updated_date).timestamp() + instance_time = timezone.localtime(db_instance.updated_date).timestamp() + if isinstance(db_instance, Project): + tasks_update = list(map(lambda db_task: timezone.localtime(db_task.updated_date).timestamp(), db_instance.tasks.all())) + instance_time = max(tasks_update + [instance_time]) if not (osp.exists(output_path) and \ - task_time <= osp.getmtime(output_path)): + instance_time <= osp.getmtime(output_path)): os.makedirs(cache_dir, exist_ok=True) with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir: temp_file = osp.join(temp_dir, 'result') - task.export_task(task_id, temp_file, dst_format, + export_fn(db_instance.id, temp_file, dst_format, server_url=server_url, save_images=save_images) os.replace(temp_file, output_path) archive_ctime = osp.getctime(output_path) scheduler = django_rq.get_scheduler() - cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL, + cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl, func=clear_export_cache, task_id=task_id, file_path=output_path, file_ctime=archive_ctime) - slogger.task[task_id].info( - "The task '{}' is exported as '{}' at '{}' " + logger.info( + "The {} '{}' is exported as '{}' at '{}' " "and available for downloading for the next {}. " "Export cache cleaning job is enqueued, id '{}'".format( - db_task.name, dst_format, output_path, CACHE_TTL, - cleaning_job.id)) + "project" if isinstance(db_instance, Project) else 'task', + db_instance.name, dst_format, output_path, cache_ttl, + cleaning_job.id + )) return output_path except Exception: - log_exception(slogger.task[task_id]) + log_exception(logger) raise def export_task_as_dataset(task_id, dst_format=None, server_url=None): - return export_task(task_id, dst_format, server_url=server_url, save_images=True) + return export(dst_format, task_id=task_id, server_url=server_url, save_images=True) def export_task_annotations(task_id, dst_format=None, server_url=None): - return export_task(task_id, dst_format, server_url=server_url, save_images=False) + return export(dst_format,task_id=task_id, server_url=server_url, save_images=False) + +def export_project_as_dataset(project_id, dst_format=None, server_url=None): + return export(dst_format, project_id=project_id, server_url=server_url, save_images=True) + + +def export_project_annotations(project_id, dst_format=None, server_url=None): + return export(dst_format, project_id=project_id, server_url=server_url, save_images=False) def clear_export_cache(task_id, file_path, file_ctime): try: @@ -116,7 +139,7 @@ def backup_task(task_id, output_path): archive_ctime = osp.getctime(output_path) scheduler = django_rq.get_scheduler() - cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL, + cleaning_job = scheduler.enqueue_in(time_delta=TASK_CACHE_TTL, func=clear_export_cache, task_id=task_id, file_path=output_path, file_ctime=archive_ctime) @@ -124,7 +147,7 @@ def backup_task(task_id, output_path): "The task '{}' is backuped at '{}' " "and available for downloading for the next {}. " "Export cache cleaning job is enqueued, id '{}'".format( - db_task.name, output_path, CACHE_TTL, + db_task.name, output_path, TASK_CACHE_TTL, cleaning_job.id)) return output_path diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 0c0b1130e115..767b393e1f3b 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -411,6 +411,8 @@ def update(self, instance, validated_data): validated_project_id = validated_data.get('project_id', None) if validated_project_id is not None and validated_project_id != instance.project_id: project = models.Project.objects.get(id=validated_data.get('project_id', None)) + if project.tasks.count() and project.tasks.first().dimension != instance.dimension: + raise serializers.ValidationError(f'Dimension ({instance.dimension}) of the task must be the same as other tasks in project ({project.tasks.first().dimension})') if instance.project_id is None: for old_label in instance.label_set.all(): try: @@ -453,8 +455,10 @@ def validate(self, attrs): # When moving task labels can be mapped to one, but when not names must be unique if 'project_id' in attrs.keys() and self.instance is not None: project_id = attrs.get('project_id') - if project_id is not None and not models.Project.objects.filter(id=project_id).count(): - raise serializers.ValidationError(f'Cannot find project with ID {project_id}') + if project_id is not None: + project = models.Project.objects.filter(id=project_id).first() + if project is None: + raise serializers.ValidationError(f'Cannot find project with ID {project_id}') # Check that all labels can be mapped new_label_names = set() old_labels = self.instance.project.label_set.all() if self.instance.project_id else self.instance.label_set.all() diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index cc6a5ffa9af2..62d39568c191 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -311,6 +311,9 @@ def _create_thread(tid, data, isImport=False): validate_dimension.set_path(upload_dir) validate_dimension.validate() + if db_task.project is not None and db_task.project.tasks.count() > 1 and db_task.project.tasks.first().dimension != validate_dimension.dimension: + raise Exception(f'Dimension ({validate_dimension.dimension}) of the task must be the same as other tasks in project ({db_task.project.tasks.first().dimension})') + if validate_dimension.dimension == models.DimensionType.DIM_3D: db_task.dimension = models.DimensionType.DIM_3D diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 3adaa017cde8..2cd724edcabb 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -298,6 +298,76 @@ def tasks(self, request, pk): return Response(serializer.data) + @swagger_auto_schema(method='get', operation_summary='Export project as a dataset in a specific format', + manual_parameters=[ + openapi.Parameter('format', openapi.IN_QUERY, + description="Desired output format name\nYou can get the list of supported formats at:\n/server/annotation/formats", + type=openapi.TYPE_STRING, required=True), + openapi.Parameter('filename', openapi.IN_QUERY, + description="Desired output file name", + type=openapi.TYPE_STRING, required=False), + openapi.Parameter('action', in_=openapi.IN_QUERY, + description='Used to start downloading process after annotation file had been created', + type=openapi.TYPE_STRING, required=False, enum=['download']) + ], + responses={'202': openapi.Response(description='Exporting has been started'), + '201': openapi.Response(description='Output file is ready for downloading'), + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), + } + ) + @action(detail=True, methods=['GET'], serializer_class=None, + url_path='dataset') + def dataset_export(self, request, pk): + db_project = self.get_object() # force to call check_object_permissions + + format_name = request.query_params.get("format", "") + return _export_annotations(db_instance=db_project, + rq_id="/api/v1/project/{}/dataset/{}".format(pk, format_name), + request=request, + action=request.query_params.get("action", "").lower(), + callback=dm.views.export_project_as_dataset, + format_name=format_name, + filename=request.query_params.get("filename", "").lower(), + ) + + @swagger_auto_schema(method='get', operation_summary='Method allows to download project annotations', + manual_parameters=[ + openapi.Parameter('format', openapi.IN_QUERY, + description="Desired output format name\nYou can get the list of supported formats at:\n/server/annotation/formats", + type=openapi.TYPE_STRING, required=True), + openapi.Parameter('filename', openapi.IN_QUERY, + description="Desired output file name", + type=openapi.TYPE_STRING, required=False), + openapi.Parameter('action', in_=openapi.IN_QUERY, + description='Used to start downloading process after annotation file had been created', + type=openapi.TYPE_STRING, required=False, enum=['download']) + ], + responses={ + '202': openapi.Response(description='Dump of annotations has been started'), + '201': openapi.Response(description='Annotations file is ready to download'), + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), + '401': openapi.Response(description='Format is not specified'), + } + ) + @action(detail=True, methods=['GET'], + serializer_class=LabeledDataSerializer) + def annotations(self, request, pk): + db_project = self.get_object() # force to call check_object_permissions + format_name = request.query_params.get('format') + if format_name: + return _export_annotations(db_instance=db_project, + rq_id="/api/v1/projects/{}/annotations/{}".format(pk, format_name), + request=request, + action=request.query_params.get("action", "").lower(), + callback=dm.views.export_project_annotations, + format_name=format_name, + filename=request.query_params.get("filename", "").lower(), + ) + else: + return Response("Format is not specified",status=status.HTTP_400_BAD_REQUEST) + class TaskFilter(filters.FilterSet): project = filters.CharFilter(field_name="project__name", lookup_expr="icontains") name = filters.CharFilter(field_name="name", lookup_expr="icontains") @@ -475,7 +545,7 @@ def retrieve(self, request, pk=None): else: return Response(status=status.HTTP_202_ACCEPTED) - ttl = dm.views.CACHE_TTL.total_seconds() + ttl = dm.views.TASK_CACHE_TTL.total_seconds() queue.enqueue_call( func=dm.views.backup_task, args=(pk, 'task_dump.zip'), @@ -679,7 +749,7 @@ def annotations(self, request, pk): if request.method == 'GET': format_name = request.query_params.get('format') if format_name: - return _export_annotations(db_task=db_task, + return _export_annotations(db_instance=db_task, rq_id="/api/v1/tasks/{}/annotations/{}".format(pk, format_name), request=request, action=request.query_params.get("action", "").lower(), @@ -806,7 +876,7 @@ def dataset_export(self, request, pk): db_task = self.get_object() # force to call check_object_permissions format_name = request.query_params.get("format", "") - return _export_annotations(db_task=db_task, + return _export_annotations(db_instance=db_task, rq_id="/api/v1/tasks/{}/dataset/{}".format(pk, format_name), request=request, action=request.query_params.get("action", "").lower(), @@ -1373,7 +1443,7 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name): return Response(status=status.HTTP_202_ACCEPTED) -def _export_annotations(db_task, rq_id, request, format_name, action, callback, filename): +def _export_annotations(db_instance, rq_id, request, format_name, action, callback, filename): if action not in {"", "download"}: raise serializers.ValidationError( "Unexpected action specified for the request") @@ -1390,9 +1460,12 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, rq_job = queue.fetch_job(rq_id) if rq_job: - last_task_update_time = timezone.localtime(db_task.updated_date) + last_instance_update_time = timezone.localtime(db_instance.updated_date) + if isinstance(db_instance, Project): + tasks_update = list(map(lambda db_task: timezone.localtime(db_task.updated_date), db_instance.tasks.all())) + last_instance_update_time = max(tasks_update + [last_instance_update_time]) request_time = rq_job.meta.get('request_time', None) - if request_time is None or request_time < last_task_update_time: + if request_time is None or request_time < last_instance_update_time: rq_job.cancel() rq_job.delete() else: @@ -1401,12 +1474,14 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, if action == "download" and osp.exists(file_path): rq_job.delete() - timestamp = datetime.strftime(last_task_update_time, + timestamp = datetime.strftime(last_instance_update_time, "%Y_%m_%d_%H_%M_%S") filename = filename or \ - "task_{}-{}-{}{}".format( - db_task.name, timestamp, - format_name, osp.splitext(file_path)[1]) + "{}_{}-{}-{}{}".format( + "project" if isinstance(db_instance, models.Project) else "task", + db_instance.name, timestamp, + format_name, osp.splitext(file_path)[1] + ) return sendfile(request, file_path, attachment=True, attachment_filename=filename.lower()) else: @@ -1427,9 +1502,9 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, except Exception: server_address = None - ttl = dm.views.CACHE_TTL.total_seconds() + ttl = (dm.views.PROJECT_CACHE_TTL if isinstance(db_instance, Project) else dm.views.TASK_CACHE_TTL).total_seconds() queue.enqueue_call(func=callback, - args=(db_task.id, format_name, server_address), job_id=rq_id, + args=(db_instance.id, format_name, server_address), job_id=rq_id, meta={ 'request_time': timezone.localtime() }, result_ttl=ttl, failure_ttl=ttl) return Response(status=status.HTTP_202_ACCEPTED) diff --git a/site/content/en/docs/manual/basics/creating_an_annotation_task.md b/site/content/en/docs/manual/basics/creating_an_annotation_task.md index 37bcd43979fe..d5e4852524a9 100644 --- a/site/content/en/docs/manual/basics/creating_an_annotation_task.md +++ b/site/content/en/docs/manual/basics/creating_an_annotation_task.md @@ -13,16 +13,19 @@ description: 'Instructions on how to create and configure an annotation task.' ## Basic configuration ### Name + The name of the task to be created. ![](/images/image005.jpg) ### Projects + The project that this task will be related with. ![](/images/image193.jpg) ### Labels + There are two ways of working with labels (available only if the task is not related to the project): - The `Constructor` is a simple way to add and adjust labels. To add a new label click the `Add label` button. @@ -67,6 +70,7 @@ description: 'Instructions on how to create and configure an annotation task.' In `Raw` and `Constructor` mode, you can press the `Copy` button to copy the list of labels. ### Select files + Press tab `My computer` to choose some files for annotation from your PC. If you select tab `Connected file share` you can choose files for annotation from your network. If you select ` Remote source` , you'll see a field where you can enter a list of URLs (one URL per line). @@ -78,63 +82,67 @@ description: 'Instructions on how to create and configure an annotation task.' ### Data formats for a 3D task To create a 3D task, you need to use the following directory structures: - {{< tabpane >}} - {{< tab header="Velodyne" >}} - VELODYNE FORMAT - Structure: - velodyne_points/ - data/ - image_01.bin - IMAGE_00 # unknown dirname, Generally image_01.png can be under IMAGE_00, IMAGE_01, IMAGE_02, IMAGE_03, etc - data/ - image_01.png - {{< /tab >}} - {{< tab header="3D pointcloud" >}} - 3D POINTCLOUD DATA FORMAT - Structure: - pointcloud/ - 00001.pcd - related_images/ - 00001_pcd/ - image_01.png # or any other image - {{< /tab >}} - {{< tab header="3D Option 1" >}} - 3D, DEFAULT DATAFORMAT Option 1 - Structure: - data/ - image.pcd - image.png - {{< /tab >}} - {{< tab header="3D Option 2" >}} - 3D, DEFAULT DATAFORMAT Option 2 - Structure: - data/ - image_1/ - image_1.pcd - context_1.png # or any other name - context_2.jpg - {{< /tab >}} - {{< /tabpane >}} + {{< tabpane >}} + {{< tab header="Velodyne" >}} + VELODYNE FORMAT + Structure: + velodyne_points/ + data/ + image_01.bin + IMAGE_00 # unknown dirname, Generally image_01.png can be under IMAGE_00, IMAGE_01, IMAGE_02, IMAGE_03, etc + data/ + image_01.png + {{< /tab >}} + {{< tab header="3D pointcloud" >}} + 3D POINTCLOUD DATA FORMAT + Structure: + pointcloud/ + 00001.pcd + related_images/ + 00001_pcd/ + image_01.png # or any other image + {{< /tab >}} + {{< tab header="3D Option 1" >}} + 3D, DEFAULT DATAFORMAT Option 1 + Structure: + data/ + image.pcd + image.png + {{< /tab >}} + {{< tab header="3D Option 2" >}} + 3D, DEFAULT DATAFORMAT Option 2 + Structure: + data/ + image_1/ + image_1.pcd + context_1.png # or any other name + context_2.jpg + {{< /tab >}} + {{< /tabpane >}} ## Advanced configuration ![](/images/image128_use_cache.jpg) ### Use zip chunks + Force to use zip chunks as compressed data. Actual for videos only. ### Use cache + Defines how to work with data. Select the checkbox to switch to the "on-the-fly data processing", which will reduce the task creation time (by preparing chunks when requests are received) and store data in a cache of limited size with a policy of evicting less popular items. See more [here](/docs/manual/advanced/data_on_fly/). ### Image Quality + Use this option to specify quality of uploaded images. The option helps to load high resolution datasets faster. Use the value from `5` (almost completely compressed images) to `100` (not compressed images). ## Overlap Size + Use this option to make overlapped segments. The option makes tracks continuous from one segment into another. Use it for interpolation mode. There are several options for using the parameter: @@ -158,22 +166,27 @@ description: 'Instructions on how to create and configure an annotation task.' even the overlap parameter isn't zero and match between corresponding shapes on adjacent segments is perfect. ### Segment size + Use this option to divide a huge dataset into a few smaller segments. For example, one job cannot be annotated by several labelers (it isn't supported). Thus using "segment size" you can create several jobs for the same annotation task. It will help you to parallel data annotation process. ### Start frame + Frame from which video in task begins. ### Stop frame + Frame on which video in task ends. ### Frame Step + Use this option to filter video frames. For example, enter `25` to leave every twenty fifth frame in the video or every twenty fifth image. ### Chunk size + Defines a number of frames to be packed in a chunk when send from client to server. Server defines automatically if empty. @@ -185,6 +198,7 @@ description: 'Instructions on how to create and configure an annotation task.' - More: 1 - 4 ### Dataset Repository + URL link of the repository optionally specifies the path to the repository for storage (`default: annotation / .zip`). The .zip and .xml file extension of annotation are supported. @@ -199,10 +213,12 @@ description: 'Instructions on how to create and configure an annotation task.' The task will be highlighted in red after creation if annotation isn't synchronized with the repository. ### Use LFS + If the annotation file is large, you can create a repository with [LFS](https://git-lfs.github.com/) support. ### Issue tracker + Specify full issue tracker's URL if it's necessary. Push `Submit` button and it will be added into the list of annotation tasks. @@ -248,4 +264,4 @@ description: 'Instructions on how to create and configure an annotation task.' --- - Push `Open` button to go to [task details](/docs/manual/basics/task-details/). +Push `Open` button to go to [task details](/docs/manual/basics/task-details/). diff --git a/tests/cypress/integration/actions_projects_models/case_94_move_task_between_projects.js b/tests/cypress/integration/actions_projects_models/case_94_move_task_between_projects.js index decc03b73742..62c3c051d711 100644 --- a/tests/cypress/integration/actions_projects_models/case_94_move_task_between_projects.js +++ b/tests/cypress/integration/actions_projects_models/case_94_move_task_between_projects.js @@ -11,16 +11,16 @@ context('Move a task between projects.', () => { label: 'car', attrName: 'color', attrVaue: 'red', - multiAttrParams: false - } + multiAttrParams: false, + }; const secondProject = { name: `Second project case ${caseID}`, label: 'bicycle', attrName: 'color', attrVaue: 'yellow', - multiAttrParams: false - } + multiAttrParams: false, + }; const taskName = `Task case ${caseID}`; const imagesCount = 1; @@ -39,14 +39,24 @@ context('Move a task between projects.', () => { const attachToProject = false; const multiAttrParams = false; - function checkTask (project, expectedResult) { + function checkTask(project, expectedResult) { cy.goToProjectsList(); cy.openProject(project); cy.get('.cvat-tasks-list-item').should(expectedResult); } before(() => { - cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, firtsProject.label, imagesCount); + cy.imageGenerator( + imagesFolder, + imageFileName, + width, + height, + color, + posX, + posY, + firtsProject.label, + imagesCount, + ); cy.createZipArchive(directoryToArchive, archivePath); cy.visit('/'); cy.login(); @@ -54,8 +64,20 @@ context('Move a task between projects.', () => { beforeEach(() => { cy.goToProjectsList(); - cy.createProjects(firtsProject.name, firtsProject.label, firtsProject.attrName, firtsProject.attrVaue, firtsProject.multiAttrParams); - cy.createProjects(secondProject.name, secondProject.label, secondProject.attrName, secondProject.attrVaue, secondProject.multiAttrParams); + cy.createProjects( + firtsProject.name, + firtsProject.label, + firtsProject.attrName, + firtsProject.attrVaue, + firtsProject.multiAttrParams, + ); + cy.createProjects( + secondProject.name, + secondProject.label, + secondProject.attrName, + secondProject.attrVaue, + secondProject.multiAttrParams, + ); cy.openProject(firtsProject.name); cy.createAnnotationTask( taskName, diff --git a/tests/cypress/integration/actions_projects_models/case_95_move_task_to_project.js b/tests/cypress/integration/actions_projects_models/case_95_move_task_to_project.js index 7ac0459dbf90..3a39367a1cc6 100644 --- a/tests/cypress/integration/actions_projects_models/case_95_move_task_to_project.js +++ b/tests/cypress/integration/actions_projects_models/case_95_move_task_to_project.js @@ -11,14 +11,14 @@ context('Move a task to a project.', () => { label: 'Tree', attrName: 'Kind', attrValue: 'Oak', - } + }; const project = { name: `Case ${caseID}`, label: 'Tree', attrName: 'Kind', - attrVaue: 'Oak' - } + attrVaue: 'Oak', + }; const imagesCount = 1; const imageFileName = `image_${task.name.replace(' ', '_').toLowerCase()}`; diff --git a/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js b/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js index b46a0c557582..ec131b67e130 100644 --- a/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js +++ b/tests/cypress/integration/actions_tasks/case_52_dump_upload_annotation.js @@ -68,9 +68,10 @@ context('Dump/Upload annotation.', { browser: '!firefox' }, () => { it('Save job. Dump annotation. Remove annotation. Save job.', () => { cy.saveJob('PATCH', 200, 'saveJobDump'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpType).click(); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').within(() => { + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.contains('button', 'OK').click(); }); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); diff --git a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js index 3d43f3d3a133..9cb52acc6029 100644 --- a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js +++ b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js @@ -66,10 +66,15 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox it('Save job. Dump annotation to YOLO format. Remove annotation. Save job.', () => { cy.saveJob('PATCH', 200, 'saveJobDump'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpType).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .trigger('wheel', {deltaY: 700}) + .contains('.cvat-modal-export-option-item', dumpType) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js index a632a4ce1ee5..c558a621cd9e 100644 --- a/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js +++ b/tests/cypress/integration/actions_tasks2/case_97_export_import_task.js @@ -67,9 +67,11 @@ context('Export, import an annotation task.', { browser: '!firefox' }, () => { .find('.cvat-item-open-task-actions > .cvat-menu-icon') .trigger('mouseover'); cy.intercept('GET', '/api/v1/tasks/**?action=export').as('exportTask'); - cy.get('.ant-dropdown').not('.ant-dropdown-hidden').within(() => { - cy.contains('[role="menuitem"]', 'Export Task').click().trigger('mouseout'); - }); + cy.get('.ant-dropdown') + .not('.ant-dropdown-hidden') + .within(() => { + cy.contains('[role="menuitem"]', new RegExp('^Export task$')).click().trigger('mouseout'); + }); cy.wait('@exportTask', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportTask').its('response.statusCode').should('equal', 201); cy.deleteTask(taskName); @@ -82,10 +84,7 @@ context('Export, import an annotation task.', { browser: '!firefox' }, () => { it('Import the task. Check id, labels, shape.', () => { cy.intercept('POST', '/api/v1/tasks?action=import').as('importTask'); - cy.get('.cvat-import-task') - .click() - .find('input[type=file]') - .attachFile(taskBackupArchiveFullName); + cy.get('.cvat-import-task').click().find('input[type=file]').attachFile(taskBackupArchiveFullName); cy.wait('@importTask', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@importTask').its('response.statusCode').should('equal', 201); cy.contains('Task has been imported succesfully').should('exist').and('be.visible'); diff --git a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js index 97fcc1d3f465..6de7a7cd3cbb 100644 --- a/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js +++ b/tests/cypress/integration/actions_tasks3/case_47_export_dataset.js @@ -6,8 +6,9 @@ import { taskName, labelName } from '../../support/const'; -context('Export as a dataset.', () => { +context('Export task dataset.', () => { const caseId = '47'; + const exportFormat = 'CVAT for images'; const rectangleShape2Points = { points: 'By 2 Points', type: 'Shape', @@ -21,16 +22,20 @@ context('Export as a dataset.', () => { before(() => { cy.openTaskJob(taskName); cy.createRectangle(rectangleShape2Points); - cy.saveJob(); + cy.saveJob('PATCH', 200, 'saveJobExportDataset'); }); describe(`Testing case "${caseId}"`, () => { - it('Go to Menu. Press "Export as a dataset" -> "CVAT for images".', () => { + it(`Go to Menu. Press "Export task dataset" with the "${exportFormat}" format.`, () => { cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDataset'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains('CVAT for images').click(); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').within(() => { + cy.get('.cvat-modal-export-select').should('contain.text', exportFormat); + cy.get('[type="checkbox"]').should('not.be.checked').check(); + cy.contains('button', 'OK').click(); }); + cy.get('.cvat-notification-notice-export-task-start').should('exist'); + cy.closeNotification('.cvat-notification-notice-export-task-start'); cy.wait('@exportDataset', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@exportDataset').its('response.statusCode').should('equal', 201); }); diff --git a/tests/cypress/integration/actions_tasks3/case_90_context_image.js b/tests/cypress/integration/actions_tasks3/case_90_context_image.js index 7171fc2ee553..02e727a40b40 100644 --- a/tests/cypress/integration/actions_tasks3/case_90_context_image.js +++ b/tests/cypress/integration/actions_tasks3/case_90_context_image.js @@ -21,7 +21,7 @@ context('Context images for 2D tasks.', () => { secondY: 450, }; - function previewRotate (directionRotation, expectedDeg) { + function previewRotate(directionRotation, expectedDeg) { if (directionRotation === 'right') { cy.get('[data-icon="rotate-right"]').click(); } else { @@ -30,30 +30,22 @@ context('Context images for 2D tasks.', () => { cy.get('.ant-image-preview-img').should('have.attr', 'style').and('contain', `rotate(${expectedDeg}deg)`); } - function previewScaleWheel (zoom, expectedScaleValue) { + function previewScaleWheel(zoom, expectedScaleValue) { cy.get('.ant-image-preview-img') - .trigger('wheel', {deltaY: zoom}) + .trigger('wheel', { deltaY: zoom }) .should('have.attr', 'style') .and('contain', `scale3d(${expectedScaleValue})`); } - function previewScaleButton (zoom, expectedScaleValue) { + function previewScaleButton(zoom, expectedScaleValue) { cy.get(`[data-icon="zoom-${zoom}"]`).click(); - cy.get('.ant-image-preview-img') - .should('have.attr', 'style') - .and('contain', `scale3d(${expectedScaleValue})`); + cy.get('.ant-image-preview-img').should('have.attr', 'style').and('contain', `scale3d(${expectedScaleValue})`); } before(() => { cy.visit('auth/login'); cy.login(); - cy.createAnnotationTask( - taskName, - labelName, - attrName, - textDefaultValue, - pathToArchive, - ); + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, pathToArchive); cy.openTaskJob(taskName); }); @@ -97,20 +89,22 @@ context('Context images for 2D tasks.', () => { }); it('Preview a context image. Move.', () => { - cy.get('.ant-image-preview-img-wrapper').should('have.attr', 'style').then((translate3d) => { - cy.get('.ant-image-preview-img').trigger('mousedown', {button: 0}); - cy.get('.ant-image-preview-moving').should('exist'); - cy.get('.ant-image-preview-wrap').trigger('mousemove', 300, 300); - cy.get('.ant-image-preview-img-wrapper').should('have.attr', 'style').and('not.equal', translate3d) - cy.get('.ant-image-preview-img').trigger('mouseup'); - cy.get('.ant-image-preview-moving').should('not.exist'); - cy.get('.ant-image-preview-img-wrapper').should('have.attr', 'style').and('equal', translate3d) - }); + cy.get('.ant-image-preview-img-wrapper') + .should('have.attr', 'style') + .then((translate3d) => { + cy.get('.ant-image-preview-img').trigger('mousedown', { button: 0 }); + cy.get('.ant-image-preview-moving').should('exist'); + cy.get('.ant-image-preview-wrap').trigger('mousemove', 300, 300); + cy.get('.ant-image-preview-img-wrapper').should('have.attr', 'style').and('not.equal', translate3d); + cy.get('.ant-image-preview-img').trigger('mouseup'); + cy.get('.ant-image-preview-moving').should('not.exist'); + cy.get('.ant-image-preview-img-wrapper').should('have.attr', 'style').and('equal', translate3d); + }); }); it('Preview a context image. Cancel preview.', () => { cy.get('.ant-image-preview-wrap').type('{Esc}'); - cy.get('.ant-image-preview-wrap').should('have.attr', 'style').and('contain', 'display: none') + cy.get('.ant-image-preview-wrap').should('have.attr', 'style').and('contain', 'display: none'); }); it('Checking issue "Context image disappears after undo/redo".', () => { diff --git a/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js b/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js index b68e1c87ac42..a87a6a2cced8 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js +++ b/tests/cypress/integration/canvas3d_functionality/case_91_canvas3d_functionality_dump_upload_annotation_point_cloud_format.js @@ -51,13 +51,14 @@ context('Canvas 3D functionality. Dump/upload annotation. "Point Cloud" format', it('Save a job. Dump with "Point Cloud" format.', () => { cy.saveJob('PATCH', 200, 'saveJob'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').then((subMenu) => { - expect(subMenu.length).to.be.equal(2); - }); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpTypePC).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpTypePC) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpTypePC); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js b/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js index d8b9676a3702..e2ef8c41e350 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js +++ b/tests/cypress/integration/canvas3d_functionality/case_92_canvas3d_functionality_dump_upload_annotation_velodyne_points_format.js @@ -51,10 +51,14 @@ context('Canvas 3D functionality. Dump/upload annotation. "Velodyne Points" form it('Save a job. Dump with "Velodyne Points" format.', () => { cy.saveJob('PATCH', 200, 'saveJob'); cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); - cy.interactMenu('Dump annotations'); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains(dumpTypeVC).click(); - }); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpTypeVC) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpTypeVC); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); cy.removeAnnotations(); diff --git a/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js b/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js index 7b8b52a1f002..1625feef8d87 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js +++ b/tests/cypress/integration/canvas3d_functionality/case_93_canvas3d_functionality_export_dataset.js @@ -15,8 +15,23 @@ context('Canvas 3D functionality. Export as a dataset.', () => { const dumpTypePC = 'Sly Point Cloud Format'; const dumpTypeVC = 'Kitti Raw Format'; + function exportDataset (format, as) { + cy.intercept('GET', '/api/v1/tasks/**/dataset**').as(as); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', format) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', format); + cy.get('.cvat-modal-export-task').find('[type="checkbox"]').should('not.be.checked').check(); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); + cy.wait(`@${as}`, { timeout: 5000 }).its('response.statusCode').should('equal', 202); + cy.wait(`@${as}`).its('response.statusCode').should('equal', 201); + } + before(() => { - cy.openTask(taskName) + cy.openTask(taskName); cy.openJob(); cy.wait(1000); // Waiting for the point cloud to display cy.create3DCuboid(cuboidCreationParams); @@ -25,23 +40,11 @@ context('Canvas 3D functionality. Export as a dataset.', () => { describe(`Testing case "${caseId}"`, () => { it('Export as a dataset with "Point Cloud" format.', () => { - cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDatasetPC'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains(dumpTypePC).click(); - }); - cy.wait('@exportDatasetPC', { timeout: 5000 }).its('response.statusCode').should('equal', 202); - cy.wait('@exportDatasetPC').its('response.statusCode').should('equal', 201); + exportDataset(dumpTypePC, 'exportDatasetPC'); }); it('Export as a dataset with "Velodyne Points" format.', () => { - cy.intercept('GET', '/api/v1/tasks/**/dataset**').as('exportDatasetVC'); - cy.interactMenu('Export as a dataset'); - cy.get('.cvat-menu-export-submenu-item').within(() => { - cy.contains(dumpTypeVC).click(); - }); - cy.wait('@exportDatasetVC', { timeout: 5000 }).its('response.statusCode').should('equal', 202); - cy.wait('@exportDatasetVC').its('response.statusCode').should('equal', 201); + exportDataset(dumpTypeVC, 'exportDatasetVC'); cy.removeAnnotations(); cy.saveJob('PUT'); }); diff --git a/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js b/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js index bfc573b6b881..5565d9068888 100644 --- a/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js +++ b/tests/cypress/integration/issues_prs2/issue_1568_cuboid_dump_annotation.js @@ -6,7 +6,7 @@ import { taskName, labelName } from '../../support/const'; -context('Dump annotation if cuboid created', () => { +context('Dump annotation if cuboid created.', () => { const issueId = '1568'; const createCuboidShape2Points = { points: 'From rectangle', @@ -17,30 +17,33 @@ context('Dump annotation if cuboid created', () => { secondX: 350, secondY: 450, }; + const dumpType = 'Datumaro'; before(() => { cy.openTaskJob(taskName); }); describe(`Testing issue "${issueId}"`, () => { - it('Create a cuboid', () => { + it('Create a cuboid.', () => { cy.createCuboid(createCuboidShape2Points); - cy.get('#cvat-objects-sidebar-state-item-1').should('contain', '1').and('contain', 'CUBOID SHAPE'); }); - it('Dump an annotation', () => { - cy.get('.cvat-annotation-header-left-group').within(() => { - cy.saveJob(); - cy.get('button').contains('Menu').trigger('mouseover', { force: true }); - }); - cy.get('.cvat-annotation-menu').within(() => { - cy.get('[title="Dump annotations"]').trigger('mouseover'); - }); - cy.get('.cvat-menu-dump-submenu-item').within(() => { - cy.contains('Datumaro').click(); - }); + + it('Dump an annotation.', () => { + cy.saveJob('PATCH', 200, `dump${dumpType}Format`); + cy.intercept('GET', '/api/v1/tasks/**/annotations**').as('dumpAnnotations'); + cy.interactMenu('Export task dataset'); + cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .contains('.cvat-modal-export-option-item', dumpType) + .click(); + cy.get('.cvat-modal-export-select').should('contain.text', dumpType); + cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); + cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); + cy.wait('@dumpAnnotations').its('response.statusCode').should('equal', 201); }); - it('Error notification is not exists', () => { - cy.wait(5000); + + it('Error notification is not exists.', () => { cy.get('.ant-notification-notice').should('not.exist'); }); }); diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 66569d26cc18..d463986d1b8e 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -101,9 +101,9 @@ Cypress.Commands.add('changeUserActiveStatus', (authKey, accountsToChangeActiveS headers: { Authorization: `Token ${authKey}`, }, - body: { - is_active: isActive, - }, + body: { + is_active: isActive, + }, }); } }); @@ -124,7 +124,6 @@ Cypress.Commands.add('checkUserStatuses', (authKey, userName, staffStatus, super expect(superuserStatus).to.be.equal(user['is_superuser']); expect(activeStatus).to.be.equal(user['is_active']); } - }); }); }); @@ -181,9 +180,7 @@ Cypress.Commands.add( } cy.contains('button', 'Submit').click(); if (expectedResult === 'success') { - cy.get('.cvat-notification-create-task-success') - .should('exist') - .find('[data-icon="close"]').click(); + cy.get('.cvat-notification-create-task-success').should('exist').find('[data-icon="close"]').click(); } if (!forProject) { cy.goToTaskList();