From fb6305fff8801e6c56a1f671eb7416258c18ede2 Mon Sep 17 00:00:00 2001 From: PMazarovich <57707673+PMazarovich@users.noreply.github.com> Date: Fri, 22 Oct 2021 16:12:42 +0300 Subject: [PATCH] Git different formats (#3708) --- cvat-ui/src/actions/tasks-actions.ts | 1 + .../advanced-configuration-form.tsx | 28 ++++++++++++++++++- .../create-task-page/create-task-content.tsx | 4 ++- .../create-task-page/create-task-page.tsx | 4 ++- cvat-ui/src/components/task-page/details.tsx | 15 ++++++++-- .../create-task-page/create-task-page.tsx | 4 ++- cvat-ui/src/utils/git-utils.ts | 7 ++++- cvat/apps/dataset_manager/formats/registry.py | 12 ++++++++ cvat/apps/dataset_repo/dataset_repo.py | 23 ++++++++------- .../migrations/0006_gitdata_format.py | 19 +++++++++++++ cvat/apps/dataset_repo/models.py | 1 + cvat/apps/dataset_repo/tests.py | 13 +++++---- cvat/apps/dataset_repo/views.py | 4 +-- 13 files changed, 110 insertions(+), 25 deletions(-) create mode 100644 cvat/apps/dataset_repo/migrations/0006_gitdata_format.py diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index 9d428a43eeb6..e5b954f6c710 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -404,6 +404,7 @@ export function createTaskAsync(data: any): ThunkAction, {}, {}, A }; gitPlugin.data.task = taskInstance; gitPlugin.data.repos = data.advanced.repository; + gitPlugin.data.format = data.advanced.format; gitPlugin.data.lfs = data.advanced.lfs; } } diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index 5ac9d3aeeac2..6cbf3860a4fb 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -6,14 +6,16 @@ import React, { RefObject } from 'react'; import { Row, Col } from 'antd/lib/grid'; import { PercentageOutlined } from '@ant-design/icons'; import Input from 'antd/lib/input'; +import Select from 'antd/lib/select'; import Checkbox from 'antd/lib/checkbox'; import Form, { FormInstance, RuleObject, RuleRender } from 'antd/lib/form'; import Text from 'antd/lib/typography/Text'; import { Store } from 'antd/lib/form/interface'; - import CVATTooltip from 'components/common/cvat-tooltip'; import patterns from 'utils/validation-patterns'; +const { Option } = Select; + export interface AdvancedConfiguration { bugTracker?: string; imageQuality?: number; @@ -23,6 +25,7 @@ export interface AdvancedConfiguration { stopFrame?: number; frameFilter?: string; lfs: boolean; + format?: string, repository?: string; useZipChunks: boolean; dataChunkSize?: number; @@ -42,6 +45,7 @@ interface Props { onSubmit(values: AdvancedConfiguration): void; installedGit: boolean; activeFileManagerTab: string; + dumpers: [] } function validateURL(_: RuleObject, value: string): Promise { @@ -276,15 +280,37 @@ class AdvancedConfigurationForm extends React.PureComponent { ); } + private renderGitFormat(): JSX.Element { + const { dumpers } = this.props; + return ( + + + + ); + } + private renderGit(): JSX.Element { return ( <> {this.renderGitRepositoryURL()} + + {this.renderGitFormat()} + {this.renderGitLFSBox()} + ); } diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index bc327f3c60c3..bb7cff45f35f 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -39,6 +39,7 @@ interface Props { taskId: number | null; projectId: number | null; installedGit: boolean; + dumpers:[] } type State = CreateTaskData; @@ -311,13 +312,14 @@ class CreateTaskContent extends React.PureComponent Advanced configuration}> diff --git a/cvat-ui/src/components/task-page/details.tsx b/cvat-ui/src/components/task-page/details.tsx index 74bf24274db1..e75bda162516 100644 --- a/cvat-ui/src/components/task-page/details.tsx +++ b/cvat-ui/src/components/task-page/details.tsx @@ -12,11 +12,11 @@ import Text from 'antd/lib/typography/Text'; import Title from 'antd/lib/typography/Title'; import moment from 'moment'; +import Descriptions from 'antd/lib/descriptions'; import getCore from 'cvat-core-wrapper'; import { getReposData, syncRepos } from 'utils/git-utils'; import { ActiveInference } from 'reducers/interfaces'; import AutomaticAnnotationProgress from 'components/tasks-page/automatic-annotation-progress'; -import Descriptions from 'antd/lib/descriptions'; import UserSelector, { User } from './user-selector'; import BugTrackerEditor from './bug-tracker-editor'; import LabelsEditorComponent from '../labels-editor/labels-editor'; @@ -39,6 +39,7 @@ interface State { subset: string; repository: string; repositoryStatus: string; + format: string; } export default class DetailsComponent extends React.PureComponent { @@ -60,6 +61,7 @@ export default class DetailsComponent extends React.PureComponent name: taskInstance.name, subset: taskInstance.subset, repository: '', + format: '', repositoryStatus: '', }; } @@ -100,6 +102,7 @@ export default class DetailsComponent extends React.PureComponent this.setState({ repository: data.url, + format: data.format, }); } }) @@ -203,7 +206,7 @@ export default class DetailsComponent extends React.PureComponent private renderDatasetRepository(): JSX.Element | boolean { const { taskInstance } = this.props; - const { repository, repositoryStatus } = this.state; + const { repository, repositoryStatus, format } = this.state; return ( !!repository && ( @@ -216,6 +219,14 @@ export default class DetailsComponent extends React.PureComponent {repository} +
+

+ Using format + {' '} + + {format} + +

{repositoryStatus === 'sync' && ( diff --git a/cvat-ui/src/containers/create-task-page/create-task-page.tsx b/cvat-ui/src/containers/create-task-page/create-task-page.tsx index 681c892dc18c..5f0f3dacadb0 100644 --- a/cvat-ui/src/containers/create-task-page/create-task-page.tsx +++ b/cvat-ui/src/containers/create-task-page/create-task-page.tsx @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -14,6 +14,7 @@ interface StateToProps { status: string; error: string; installedGit: boolean; + dumpers:[] } interface DispatchToProps { @@ -31,6 +32,7 @@ function mapStateToProps(state: CombinedState): StateToProps { return { ...creates, installedGit: state.plugins.list.GIT_INTEGRATION, + dumpers: state.formats.annotationFormats.dumpers, }; } diff --git a/cvat-ui/src/utils/git-utils.ts b/cvat-ui/src/utils/git-utils.ts index 84329db8764d..6c7cde68de2b 100644 --- a/cvat-ui/src/utils/git-utils.ts +++ b/cvat-ui/src/utils/git-utils.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -22,6 +22,7 @@ interface GitPlugin { }; }; data: { + format: any; task: any; lfs: boolean; repos: string; @@ -37,6 +38,7 @@ interface ReposData { value: 'sync' | '!sync' | 'merged'; error: string | null; }; + format: string } function waitForClone(cloneResponse: any): Promise { @@ -93,6 +95,7 @@ async function cloneRepository(this: any, plugin: GitPlugin, createdTask: any): data: JSON.stringify({ path: plugin.data.repos, lfs: plugin.data.lfs, + format: plugin.data.format, tid: createdTask.id, }), }) @@ -127,6 +130,7 @@ export function registerGitPlugin(): void { data: { task: null, lfs: false, + format: '', repos: '', }, callbacks: { @@ -152,6 +156,7 @@ export async function getReposData(tid: number): Promise { value: response.status.value, error: response.status.error, }, + format: response.format, }; } diff --git a/cvat/apps/dataset_manager/formats/registry.py b/cvat/apps/dataset_manager/formats/registry.py index 959127ca0176..2d8de6296fe1 100644 --- a/cvat/apps/dataset_manager/formats/registry.py +++ b/cvat/apps/dataset_manager/formats/registry.py @@ -51,7 +51,19 @@ def __call__(self, *args, **kwargs): return target + EXPORT_FORMATS = {} + + +def format_for(export_format, mode): + format_name = export_format + if mode == "annotation": + format_name = "CVAT for images 1.1" + elif export_format not in EXPORT_FORMATS: + format_name = "CVAT for video 1.1" + return format_name + + def exporter(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D): assert name not in EXPORT_FORMATS, "Export format '%s' already registered" % name def wrap_with_params(f_or_cls): diff --git a/cvat/apps/dataset_repo/dataset_repo.py b/cvat/apps/dataset_repo/dataset_repo.py index d86553f50b4e..b6ba56a9d163 100644 --- a/cvat/apps/dataset_repo/dataset_repo.py +++ b/cvat/apps/dataset_repo/dataset_repo.py @@ -9,7 +9,6 @@ import re import shutil import subprocess -from glob import glob import zipfile import django_rq @@ -17,11 +16,12 @@ from django.db import transaction from django.utils import timezone +from cvat.apps.dataset_manager.formats.registry import format_for from cvat.apps.dataset_manager.task import export_task +from cvat.apps.dataset_repo.models import GitData, GitStatusChoice from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Job, Task, User from cvat.apps.engine.plugins import add_plugin -from cvat.apps.dataset_repo.models import GitData, GitStatusChoice def _have_no_access_exception(ex): @@ -66,6 +66,7 @@ def __init__(self, db_git, db_task, user): self._branch_name = 'cvat_{}_{}'.format(db_task.id, self._task_name) self._annotation_file = os.path.join(self._cwd, self._path) self._sync_date = db_git.sync_date + self._format = db_git.format self._lfs = db_git.lfs @@ -265,16 +266,13 @@ def push(self, user, scheme, host, db_task, last_save): # Dump an annotation timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - if self._task_mode == "annotation": - format_name = "CVAT for images 1.1" - else: - format_name = "CVAT for video 1.1" dump_name = os.path.join(db_task.get_task_dirname(), - "git_annotation_{}.zip".format(timestamp)) + "git_annotation_{}_{}.zip".format(self._format, timestamp)) + export_task( task_id=self._tid, dst_file=dump_name, - format_name=format_name, + format_name=self._format, server_url=scheme + host, save_images=False, ) @@ -353,7 +351,7 @@ def remote_status(self, last_save): return GitStatusChoice.NON_SYNCED -def initial_create(tid, git_path, lfs, user): +def initial_create(tid, git_path, export_format, lfs, user): try: db_task = Task.objects.get(pk = tid) path_pattern = r"\[(.+)\]" @@ -373,9 +371,12 @@ def initial_create(tid, git_path, lfs, user): if len(_split) < 2 or _split[1] not in [".xml", ".zip"]: raise Exception("Only .xml and .zip formats are supported") + format_name = format_for(export_format, db_task.mode) + db_git = GitData() db_git.url = git_path db_git.path = path + db_git.format = format_name db_git.task = db_task db_git.lfs = lfs @@ -416,7 +417,7 @@ def get(tid, user): response = {} response["url"] = {"value": None} response["status"] = {"value": None, "error": None} - + response["format"] = {"format": None} db_task = Task.objects.get(pk = tid) if GitData.objects.filter(pk = db_task).exists(): db_git = GitData.objects.select_for_update().get(pk = db_task) @@ -428,12 +429,14 @@ def get(tid, user): if rq_job is not None and (rq_job.is_queued or rq_job.is_started): db_git.status = GitStatusChoice.SYNCING response['status']['value'] = str(db_git.status) + response['format'] = str(db_git.format) else: try: _git = Git(db_git, db_task, user) _git.init_repos(True) db_git.status = _git.remote_status(db_task.updated_date) response['status']['value'] = str(db_git.status) + response['format'] = str(db_git.format) except git.exc.GitCommandError as ex: _have_no_access_exception(ex) db_git.save() diff --git a/cvat/apps/dataset_repo/migrations/0006_gitdata_format.py b/cvat/apps/dataset_repo/migrations/0006_gitdata_format.py new file mode 100644 index 000000000000..350810f746cc --- /dev/null +++ b/cvat/apps/dataset_repo/migrations/0006_gitdata_format.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.3 on 2019-02-05 17:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('dataset_repo', '0005_auto_20201019_1100'), + ] + + replaces = [('git', '0006_gitdata_format')] + + operations = [ + migrations.AddField( + model_name='gitdata', + name='format', + field=models.CharField(max_length=256) + ), + ] diff --git a/cvat/apps/dataset_repo/models.py b/cvat/apps/dataset_repo/models.py index 07a6ece6d50d..0249fdc80367 100644 --- a/cvat/apps/dataset_repo/models.py +++ b/cvat/apps/dataset_repo/models.py @@ -20,6 +20,7 @@ class GitData(models.Model): task = models.OneToOneField(Task, on_delete = models.CASCADE, primary_key = True) url = models.URLField(max_length = 2000) path = models.CharField(max_length=256) + format = models.CharField(max_length=256) sync_date = models.DateTimeField(auto_now_add=True) status = models.CharField(max_length=20, default=GitStatusChoice.NON_SYNCED) lfs = models.BooleanField(default=True) diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index 4a397f4433b7..a2261a4c183c 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -23,7 +23,7 @@ orig_execute = git.cmd.Git.execute GIT_URL = "https://1.2.3.4/repo/exist.git" PARSE_GIT_URL = "git@1.2.3.4:repo/exist.git" - +EXPORT_FORMAT = "CVAT for video 1.1" def generate_image_file(filename, size=(100, 50)): f = BytesIO() @@ -414,6 +414,7 @@ def test_push(self): db_task = Task.objects.get(pk=task["id"]) db_git = GitData() db_git.url = GIT_URL + db_git.format = EXPORT_FORMAT db_git.path = "annotation.zip" db_git.sync_date = timezone.now() @@ -429,7 +430,7 @@ def test_push(self): @mock.patch('git.Repo.clone_from', new=GitRepo.clone) def test_request_initial_create(self): task = self._create_task(init_repos=False) - initial_create(task["id"], GIT_URL, 1, self.user) + initial_create(task["id"], GIT_URL, EXPORT_FORMAT, 1, self.user) self.assertTrue(osp.isdir(osp.join(task["repos_path"], '.git'))) git_repo = git.Repo(task["repos_path"]) with git_repo.config_reader() as cw: @@ -444,7 +445,7 @@ def test_request_initial_create(self): def test_request_push(self): task = self._create_task(init_repos=False) tid = task["id"] - initial_create(tid, GIT_URL, 1, self.user) + initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user) self.add_file(task["repos_path"], "file.txt") push(tid, self.user, "", "") @@ -459,7 +460,7 @@ def test_request_push(self): def test_push_and_request_get(self): task = self._create_task(init_repos=False) tid = task["id"] - initial_create(tid, GIT_URL, 1, self.user) + initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user) self.add_file(task["repos_path"], "file.txt") push(tid, self.user, "", "") response = get(tid, self.user) @@ -474,7 +475,7 @@ def test_push_and_request_get(self): def test_request_get(self): task = self._create_task(init_repos=False) tid = task["id"] - initial_create(tid, GIT_URL, 1, self.user) + initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user) response = get(tid, self.user) self.assertTrue(response["status"]["value"], "not sync") @@ -487,7 +488,7 @@ def test_request_get(self): def test_request_on_save(self): task = self._create_task(init_repos=False) tid = task["id"] - initial_create(tid, GIT_URL, 1, self.user) + initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user) jobs = self._get_jobs(tid) ann = { diff --git a/cvat/apps/dataset_repo/views.py b/cvat/apps/dataset_repo/views.py index dd50268ce5da..fae92d6de709 100644 --- a/cvat/apps/dataset_repo/views.py +++ b/cvat/apps/dataset_repo/views.py @@ -40,14 +40,14 @@ def check_process(request, rq_id): def create(request, tid): try: slogger.task[tid].info("create repository request") - body = json.loads(request.body.decode('utf-8')) path = body["path"] + export_format = body["format"] lfs = body["lfs"] rq_id = "git.create.{}".format(tid) queue = django_rq.get_queue("default") - queue.enqueue_call(func = CVATGit.initial_create, args = (tid, path, lfs, request.user), job_id = rq_id) + queue.enqueue_call(func = CVATGit.initial_create, args = (tid, path, export_format, lfs, request.user), job_id = rq_id) return JsonResponse({ "rq_id": rq_id }) except Exception as ex: slogger.glob.error("error occurred during initial cloning repository request with rq id {}".format(rq_id), exc_info=True)