From 611cbfe9b331b00ea0c40efdb1b14bd729ce1187 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 25 Oct 2018 11:48:28 +0300 Subject: [PATCH 01/84] Initial commit --- cvat/apps/git/__init__.py | 3 +++ cvat/apps/git/admin.py | 7 +++++++ cvat/apps/git/apps.py | 9 +++++++++ cvat/apps/git/migrations/__init__.py | 3 +++ cvat/apps/git/models.py | 7 +++++++ cvat/apps/git/tests.py | 7 +++++++ cvat/apps/git/views.py | 7 +++++++ 7 files changed, 43 insertions(+) create mode 100644 cvat/apps/git/__init__.py create mode 100644 cvat/apps/git/admin.py create mode 100644 cvat/apps/git/apps.py create mode 100644 cvat/apps/git/migrations/__init__.py create mode 100644 cvat/apps/git/models.py create mode 100644 cvat/apps/git/tests.py create mode 100644 cvat/apps/git/views.py diff --git a/cvat/apps/git/__init__.py b/cvat/apps/git/__init__.py new file mode 100644 index 000000000000..b66dde17a5cf --- /dev/null +++ b/cvat/apps/git/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/cvat/apps/git/admin.py b/cvat/apps/git/admin.py new file mode 100644 index 000000000000..6f067d6c04ee --- /dev/null +++ b/cvat/apps/git/admin.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.contrib import admin + +# Register your models here. diff --git a/cvat/apps/git/apps.py b/cvat/apps/git/apps.py new file mode 100644 index 000000000000..9ef815879479 --- /dev/null +++ b/cvat/apps/git/apps.py @@ -0,0 +1,9 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.apps import AppConfig + + +class GitConfig(AppConfig): + name = 'git' diff --git a/cvat/apps/git/migrations/__init__.py b/cvat/apps/git/migrations/__init__.py new file mode 100644 index 000000000000..b66dde17a5cf --- /dev/null +++ b/cvat/apps/git/migrations/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/cvat/apps/git/models.py b/cvat/apps/git/models.py new file mode 100644 index 000000000000..acafafb51f52 --- /dev/null +++ b/cvat/apps/git/models.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.db import models + +# Create your models here. diff --git a/cvat/apps/git/tests.py b/cvat/apps/git/tests.py new file mode 100644 index 000000000000..3517aa28fb20 --- /dev/null +++ b/cvat/apps/git/tests.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.test import TestCase + +# Create your tests here. diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py new file mode 100644 index 000000000000..4d915355dc0b --- /dev/null +++ b/cvat/apps/git/views.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.shortcuts import render + +# Create your views here. From 35a885ad434084f7ccd35a73fa7fe8b06c211c89 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 25 Oct 2018 17:02:24 +0300 Subject: [PATCH 02/84] UI --- .../static/dashboard/js/dashboard.js | 2 +- .../engine/static/engine/js/annotationUI.js | 5 +- cvat/apps/git/__init__.py | 5 + cvat/apps/git/migrations/0001_initial.py | 23 ++ cvat/apps/git/models.py | 6 +- .../apps/git/static/git/js/dashboardPlugin.js | 46 ++++ cvat/apps/git/static/git/js/enginePlugin.js | 212 ++++++++++++++++++ cvat/apps/git/urls.py | 15 ++ cvat/apps/git/views.py | 106 ++++++++- cvat/settings/base.py | 3 + cvat/urls.py | 3 + 11 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 cvat/apps/git/migrations/0001_initial.py create mode 100644 cvat/apps/git/static/git/js/dashboardPlugin.js create mode 100644 cvat/apps/git/static/git/js/enginePlugin.js create mode 100644 cvat/apps/git/urls.py diff --git a/cvat/apps/dashboard/static/dashboard/js/dashboard.js b/cvat/apps/dashboard/static/dashboard/js/dashboard.js index 091ed66827dd..4dbd60cbcd33 100644 --- a/cvat/apps/dashboard/static/dashboard/js/dashboard.js +++ b/cvat/apps/dashboard/static/dashboard/js/dashboard.js @@ -445,7 +445,7 @@ function createTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, on done = true; clearInterval(requestInterval); onComplete(); - onSuccessCreate(); + onSuccessCreate(tid); } else if (data['state'] == 'error') { done = true; diff --git a/cvat/apps/engine/static/engine/js/annotationUI.js b/cvat/apps/engine/static/engine/js/annotationUI.js index 53a0edca03f3..db757bc7b75b 100644 --- a/cvat/apps/engine/static/engine/js/annotationUI.js +++ b/cvat/apps/engine/static/engine/js/annotationUI.js @@ -57,6 +57,9 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) { id: job.jobid, images: job.image_meta_data, }, + task: { + id: job.taskid + }, search: { value: window.location.search, @@ -204,7 +207,7 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) { $(window).on('click', function(event) { Logger.updateUserActivityTimer(); - if (['helpWindow', 'settingsWindow'].indexOf(event.target.id) != -1) { + if (event.target.classList.contains('modal')) { event.target.classList.add('hidden'); } }); diff --git a/cvat/apps/git/__init__.py b/cvat/apps/git/__init__.py index b66dde17a5cf..5c84ffca484b 100644 --- a/cvat/apps/git/__init__.py +++ b/cvat/apps/git/__init__.py @@ -1,3 +1,8 @@ # Copyright (C) 2018 Intel Corporation # # SPDX-License-Identifier: MIT + +from cvat.settings.base import JS_3RDPARTY + +JS_3RDPARTY['dashboard'] = JS_3RDPARTY.get('dashboard', []) + ['git/js/dashboardPlugin.js'] +JS_3RDPARTY['engine'] = JS_3RDPARTY.get('engine', []) + ['git/js/enginePlugin.js'] diff --git a/cvat/apps/git/migrations/0001_initial.py b/cvat/apps/git/migrations/0001_initial.py new file mode 100644 index 000000000000..c7a64c0fb07e --- /dev/null +++ b/cvat/apps/git/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# Generated by Django 2.0.9 on 2018-10-25 13:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('engine', '0012_auto_20181025_1618'), + ] + + operations = [ + migrations.CreateModel( + name='GitData', + fields=[ + ('task', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='engine.Task')), + ('url', models.URLField(max_length=2000)), + ], + ), + ] diff --git a/cvat/apps/git/models.py b/cvat/apps/git/models.py index acafafb51f52..3288a3cebe1b 100644 --- a/cvat/apps/git/models.py +++ b/cvat/apps/git/models.py @@ -3,5 +3,9 @@ # SPDX-License-Identifier: MIT from django.db import models +from cvat.apps.engine.models import Task -# Create your models here. + +class GitData(models.Model): + task = models.OneToOneField(Task, on_delete = models.CASCADE, primary_key = True) + url = models.URLField(max_length = 2000) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js new file mode 100644 index 000000000000..035c5ca56345 --- /dev/null +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +"use strict"; + +document.addEventListener("DOMContentLoaded", () => { + $(` + + + + + `).insertAfter($("#dashboardBugTrackerInput").parent().parent()); + + let originalCreateTaskRequest = createTaskRequest; + + window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) { + let originalOnSuccessCreate = onSuccessCreate; + + onSuccessCreate = (tid) => { + let gitURL = $('#gitAnnoReposInput').prop('value'); + + if (gitURL.length) { + $.ajax({ + type: 'POST', + url: '/git/repository/create', + data: JSON.stringify({ + 'tid': tid, + 'url': gitURL, + }), + contentType: 'application/json;charset=utf-8', + error: (data) => { + throw Error(`Warning: Can't create git record for task ${tid}: ${data.responseText}`); + }, + complete: () => { + originalOnSuccessCreate(); + } + }); + } + } + + originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); + } +}); diff --git a/cvat/apps/git/static/git/js/enginePlugin.js b/cvat/apps/git/static/git/js/enginePlugin.js new file mode 100644 index 000000000000..84d132ca521d --- /dev/null +++ b/cvat/apps/git/static/git/js/enginePlugin.js @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2018 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +"use strict"; + +document.addEventListener("DOMContentLoaded", () => { + let gitReposWindowID = 'gitReposWindow'; + let closeReposWindowButtonId = 'closeGitReposButton'; + let gitRepositoryURLInputTextId = 'gitReposInputText'; + let gitRepositoryURLUpdateButtonId = 'gitReposUpdateButton'; + let gitRepositoryPushButtonId = 'gitReposPushButton'; + let gitLabelStatusId = 'gitReposLabelStatus'; + let gitLabelMessageId = 'gitReposLabelMessage'; + + $(` + `).appendTo('#taskAnnotationCenterPanel'); + + let gitWindow = $(`#${gitReposWindowID}`); + + $(``).on('click', () => { + $('#annotationMenu').addClass('hidden'); + gitWindow.removeClass('hidden'); + updateRepositoryInfo(); + }).prependTo($('#engineMenuButtons')); + + /* Used unicode characters: + updating: ⏲ + pushed: ✓ + actual: ★ + obsolete: ☆ + error: ⚠ + */ + + let closeRepositoryWindowButton = $(`#${closeReposWindowButtonId}`); + let repositoryURLInput = $(`#${gitRepositoryURLInputTextId}`); + let gitLabelMessage = $(`#${gitLabelMessageId}`); + let gitLabelStatus = $(`#${gitLabelStatusId}`); + let repositoryUpdateButton = $(`#${gitRepositoryURLUpdateButtonId}`); + let repositoryPushButton = $(`#${gitRepositoryPushButtonId}`); + + repositoryURLInput.on('keyup keydown keypress', (e) => e.stopPropagation()); + + closeRepositoryWindowButton.on('click', () => { + gitWindow.addClass('hidden'); + }); + + repositoryUpdateButton.on('click', () => { + let gitURL = repositoryURLInput.prop('value').replace(/\s/g,''); + if (!gitURL) { + removeGitURL(); + } + else { + getGitURL((data) => { + if (!data.url.value) { + createGitURL(gitURL); + } + else { + updateGitURL(gitURL); + } + }, () => { + let message = `Get git URL errors. Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + }); + } + }); + + repositoryPushButton.on('click', () => { + // to do + }); + + function createGitURL(url) { + $.ajax({ + url: '/git/repository/create', + type: 'POST', + data: JSON.stringify({ + 'tid': window.cvat.task.id, + 'url': url, + }), + success: () => { + updateRepositoryInfo(); + }, + error: (data) => { + let message = `Can not update repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + } + + function updateGitURL(url) { + $.ajax({ + url: '/git/repository/update', + type: 'POST', + data: JSON.stringify({ + 'jid': window.cvat.job.id, + 'url': url, + }), + success: () => { + updateRepositoryInfo(); + }, + error: (data) => { + let message = `Can not update repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + } + + function removeGitURL() { + $.ajax({ + url: '/git/repository/delete/' + window.cvat.job.id, + type: 'DELETE', + success: () => { + updateRepositoryInfo(); + }, + error: (data) => { + let message = `Can not delete repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + } + + function getGitURL(success, error) { + $.ajax({ + url: '/git/repository/get/' + window.cvat.job.id, + type: 'GET', + success: success, + error: error + }); + } + + function updateRepositoryInfo() { + gitLabelMessage.css('color', 'yellowgreen').text('Updating..'); + gitLabelStatus.css('color', 'yellowgreen').text('\u23F2'); + + getGitURL((data) => { + if (!data.url.value) { + gitLabelMessage.css('color', 'black').text('Repository is not attached'); + repositoryURLInput.prop('value', 'Repository is not attached'); + return; + } + + repositoryURLInput.prop('value', data.url.value); + + if (!data.status.value) { + gitLabelStatus.css('color', 'red').text('\u26a0'); + gitLabelMessage.css('color', 'red').text(data.status.error); + return; + } + + if (data.status.value == "actual") { + gitLabelStatus.css('color', 'darkgreen').text('\u2605'); + gitLabelMessage.css('color', 'darkgreen').text('Repository contains actual data'); + } + else if (data.status.value == "obsolete") { + gitLabelStatus.css('color', 'darkgreen').text('\u2606'); + gitLabelMessage.css('color', 'black').text('Repository contains obsolete data'); + } + else { + let message = `Got unknown repository status: ${data.status.value}`; + gitLabelStatus.css('color', 'red').text('\u26a0'); + gitLabelMessage.css('color', 'red').text(message); + throw Error(message); + } + }, (data) => { + gitWindow.addClass('hidden'); + let message = `Get respository URL errors. Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + }); + } +}); diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py new file mode 100644 index 000000000000..5e816c6fbf36 --- /dev/null +++ b/cvat/apps/git/urls.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from django.urls import path +from . import views + + +urlpatterns = [ + path('create', views.create_repository), + path('update', views.update_repository), + path('get/', views.get_repository), + path('delete/', views.delete_repository), +] \ No newline at end of file diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 4d915355dc0b..25bb04804892 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -3,5 +3,109 @@ # SPDX-License-Identifier: MIT from django.shortcuts import render +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse +from django.contrib.auth.decorators import permission_required -# Create your views here. +from cvat.apps.authentication.decorators import login_required +from cvat.apps.engine.log import slogger +from cvat.apps.engine.models import Task, Job +from cvat.apps.git.models import GitData + +import json + + +@login_required +@permission_required('engine.add_task', raise_exception=True) +def create_repository(request): + try: + data = json.loads(request.body.decode('utf-8')) + tid = data['tid'] + url = data['url'] + + db_task = Task.objects.get(pk = tid) + if GitData.objects.filter(pk = db_task).exists(): + raise Exception('git repository for task already exists') + + db_git = GitData() + db_git.task = db_task + db_git.url = url + db_git.save() + except Exception as e: + slogger.glob.error("cannot create git repository for task #{}".format(tid), exc_info=True) + return HttpResponseBadRequest(str(e)) + + return HttpResponse() + + +@login_required +@permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) +def update_repository(request): + try: + data = json.loads(request.body.decode('utf-8')) + jid = data['jid'] + url = data['url'] + + db_job = Job.objects.select_related('segment__task').get(pk = jid) + + db_git = GitData.objects.select_for_update().get(pk = db_job.segment.task) + db_git.url = url + db_git.save() + except Exception as e: + try: + slogger.job[jid].error("can not update git repository", exc_info=True) + except: + pass + + return HttpResponseBadRequest(str(e)) + + return HttpResponse() + + +@login_required +@permission_required(perm=['engine.view_task'], raise_exception=True) +def get_repository(request, jid): + try: + response = { + 'url': { + 'value': None, + }, + 'status': { + 'value': None, + 'error': None + } + } + + db_job = Job.objects.select_related("segment__task").get(pk = jid) + if not GitData.objects.filter(pk = db_job.segment.task).exists(): + return JsonResponse(response) + + response['url']['value'] = GitData.objects.get(pk = db_job.segment.task).url + response['status']['error'] = 'not implemented' + + return JsonResponse(response) + except Exception as e: + try: + slogger.job[jid].error("can not get git repository info", exc_info=True) + except: + pass + return HttpResponseBadRequest(str(e)) + + +@login_required +@permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) +def delete_repository(request, jid): + try: + db_job = Job.objects.select_related("segment__task").get(pk = jid) + + if GitData.objects.filter(pk = db_job.segment.task).exists(): + db_git = GitData.objects.select_for_update().get(pk = db_job.segment.task) + db_git.delete() + + except Exception as e: + try: + slogger.job[jid].error("can not delete git repository data", exc_info=True) + except: + pass + return HttpResponseBadRequest(str(e)) + + return HttpResponse() diff --git a/cvat/settings/base.py b/cvat/settings/base.py index bcb4f1cbd856..00112a70b921 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -60,6 +60,9 @@ if 'yes' == os.environ.get('TF_ANNOTATION', 'no'): INSTALLED_APPS += ['cvat.apps.tf_annotation'] +if 'yes' == os.environ.get('GIT_SUPPORT', 'no'): + INSTALLED_APPS += ['cvat.apps.git'] + if os.getenv('DJANGO_LOG_VIEWER_HOST'): INSTALLED_APPS += ['cvat.apps.log_viewer'] diff --git a/cvat/urls.py b/cvat/urls.py index 4b6eecf84ea2..e054475ffb3d 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -37,6 +37,9 @@ if apps.is_installed('cvat.apps.tf_annotation'): urlpatterns.append(path('tensorflow/annotation/', include('cvat.apps.tf_annotation.urls'))) +if apps.is_installed('cvat.apps.git'): + urlpatterns.append(path('git/repository/', include('cvat.apps.git.urls'))) + if apps.is_installed('cvat.apps.log_viewer'): urlpatterns.append(path('analytics/', include('cvat.apps.log_viewer.urls'))) From 0a662c947bdad71af2acb2c8ae8455deeb648d5b Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 29 Oct 2018 16:58:08 +0300 Subject: [PATCH 03/84] Initial code with logic --- cvat/apps/git/git.py | 169 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 cvat/apps/git/git.py diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py new file mode 100644 index 000000000000..af60a7ea1374 --- /dev/null +++ b/cvat/apps/git/git.py @@ -0,0 +1,169 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +#!/bin/bash + +import datetime +import shutil +import json +import git +import re +import os + +DIFF_DIR = "cvat_diffs" +CVAT_USER = "cvat" +CVAT_EMAIL = "cvat@example.com" + + +class Git: + self.__url = None + self.__tid = None + self.__user = None + self.__cwd = None + self.__rep = None + + + def __init__(self, url, tid, user): + self.__url = url + self.__tid = tid + self.__user = user + self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repository") + + try: + self.__rep = git.Repo(self.__cwd) + except git.exc.InvalidGitRepositoryError: + shutil.rmtree(self.__cwd, True) + self.reclone() + except git.exc.NoSuchPathError: + self.clone() + + + def _parse_url(self): + http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)" + ssh_pattern = "([a-zA-Z0-9._-]+)@([a-zA-Z0-9._-]+):([a-zA-Z.-]+)/([a-zA-Z.-]+)" + + http_match = re.match(http_pattern, self.__url) + ssh_match = re.match(ssh_pattern, self.__url) + + if http_match: + print("{} is http URL".format(self.__url)) + user = "git" + scheme = http_match.group(1) if http_match.group(1) else "https" + host = http_match.group(2) + repos = "{}/{}".format(http_match.group(3), http_match.group(4)) + if not repos.endswith(".git"): + repos += ".git" + return scheme, user, host, repos + elif ssh_match: + print("{} is ssh URL".format(self.__url)) + scheme = "https" + user = ssh_match.group(1) + host = ssh_match.group(2) + repos = "{}/{}".format(ssh_match.group(3), ssh_match.group(4)) + if not repos.endswith(".git"): + repos += ".git" + return scheme, user, host, repos + else: + raise Exception("Couldn't parse URL") + + + def ssh_url(self): + scheme, user, host, repos = self._parse_url() + return "{}@{}:{}".format(user, host, repos) + + + def http_url(self): + scheme, user, host, repos = self._parse_url() + return "{}://{}/{}".format(scheme, host, repos) + + + def clone(self): + os.makedirs(self.__cwd) + ssh_url = self.ssh_url() + self.__rep = git.Repo.clone_from(ssh_url, self.__cwd) + + with self.__rep.config_writer() as cw: + if not cw.has_section("user"): + cw.add_section("user") + cw.set("user", "name", CVAT_USER) + cw.set("user", "email", CVAT_EMAIL) + cw.release() + + if not len(rep.heads): + readme_md_name = os.path.join(self.__cwd, "README.md") + with open(readme_md_name, "w"): + pass + rep.index.add([readme_md_name]) + rep.create_head("master") + rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) + rep.git.push("origin", branch_name) + try: + rep.git.push("origin", "master") + except git.exc.GitError: + print("Master branch wasn't found, but script couldn't create it") + + os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) + gitignore = os.path.join(self.__cwd, ".gitignore") + file_mode = "a" if os.path.isfile(os.path.join(self.__cwd, ".gitignore")) else "w" + with open(gitignore, file_mode) as gitignore: + gitignore.writelines(["\n", "{}/".format(DIFF_DIR), "\n"]) + + + def reclone(self): + if os.path.exists(self.__cwd): + if not os.path.isdir(self.__cwd): + os.remove(self.__cwd) + else: + tmp_repo = os.path.join(os.path.split(self.__cwd)[:-1], "tmp_repo".format(self.__user)) + os.rename(self.__cwd, tmp_repo) + successful_cloning = False + + try: + self.clone() + successful_cloning = True + except git.exc.GitError: + os.rename(tmp_repo, self.__cwd) + raise Exception("Couldn't clone repository") + + if successful_cloning: + if os.path.exists(os.path.join(tmp_repo, DIFF_DIR)): + diffs_to_move = list(map(lambda x: os.path.join(DIFF_DIR, x), os.listdir(os.path.join(tmp_repo, DIFF_DIR)))) + diffs_to_move = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff"), diffs_to_move) + + for diff in diffs_to_move: + os.rename(os.path.join(tmp_repo, diff), os.path.join(self.__cwd, diff)) + shutil.rmtree(tmp_repo, True) + return + + self.clone() + + + def onsave(self, changes_dict): + diff_dir = os.path.join(self.__cwd, DIFF_DIR) + os.makedirs(diff_dir, exist_ok=True) + diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + last_num = 0 + for f in diff_files: + number = f.split("_")[0] + number = int(number) if number.isdigit() else last_num + last_num = number + + with open("{}_{}.diff".format(last_num + 1, self.__user), 'w') as f: + f.write(json.dumps(changes_dict)) + + + def pull(self): + pass + + + def push(self): + pass + + + def remote_status(self): + diff_dir = os.path.join(self.__cwd, DIFF_DIR) + os.makedirs(diff_dir, exist_ok=True) + diffs = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs)) + return "actual" if not len(diffs) else "obsolete" From dd7bc655e2b119ad5738cf2ae3a9d9883ee38ea5 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Oct 2018 12:17:23 +0300 Subject: [PATCH 04/84] Debug push and pull methods --- cvat/apps/git/git.py | 71 +++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index af60a7ea1374..aede86ad7cae 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -17,11 +17,11 @@ class Git: - self.__url = None - self.__tid = None - self.__user = None - self.__cwd = None - self.__rep = None + __url = None + __tid = None + __user = None + __cwd = None + __rep = None def __init__(self, url, tid, user): @@ -67,14 +67,25 @@ def _parse_url(self): else: raise Exception("Couldn't parse URL") + def _create_master(self): + if not len(self.__rep.heads): + readme_md_name = os.path.join(self.__cwd, "README.md") + with open(readme_md_name, "w"): + pass + self.__rep.index.add([readme_md_name]) + self.__rep.head.reference = self.__rep.create_head("master") + self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) + def ssh_url(self): - scheme, user, host, repos = self._parse_url() + user, host, repos = self._parse_url()[1:] return "{}@{}:{}".format(user, host, repos) def http_url(self): - scheme, user, host, repos = self._parse_url() + data = self._parse_url() + scheme = data[0] + host, repos = data[2:] return "{}://{}/{}".format(scheme, host, repos) @@ -90,18 +101,11 @@ def clone(self): cw.set("user", "email", CVAT_EMAIL) cw.release() - if not len(rep.heads): - readme_md_name = os.path.join(self.__cwd, "README.md") - with open(readme_md_name, "w"): - pass - rep.index.add([readme_md_name]) - rep.create_head("master") - rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) - rep.git.push("origin", branch_name) - try: - rep.git.push("origin", "master") - except git.exc.GitError: - print("Master branch wasn't found, but script couldn't create it") + self._create_master() + try: + self.__rep.git.push("origin", "master") + except git.exc.GitError: + print("Remote master branch wasn't found, but script couldn't push it") os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) gitignore = os.path.join(self.__cwd, ".gitignore") @@ -154,11 +158,36 @@ def onsave(self, changes_dict): def pull(self): - pass + if "master" not in self.__rep.heads: + self._create_master() + + remote_branches = [] + for remote_branch in self.__rep.git.branch("-r").split("\n")[1:]: + remote_branches.append(remote_branch.split("/")[-1]) + + if "master" in remote_branches: + try: + self.__rep.git.pull("origin", "master") + except git.exc.GitError: + self.reclone() def push(self): - pass + self.pull() + if CVAT_USER in list(map(lambda x: x.name, self.__rep.heads)): + git.Head.delete(self.__rep.heads[CVAT_USER].repo, self.__rep.heads[CVAT_USER]) + self.__rep.create_head(CVAT_USER) + + os.makedirs(os.path.join(self.__cwd, 'annotation'), exist_ok = True) + with open(os.path.join(self.__cwd, 'annotation', 'SomeTask.dump'), 'w'): + # create fake dump file + pass + + self.__rep.index.add([os.path.join(self.__cwd, 'annotation', '*.dump')]) + self.__rep.git.push("origin", CVAT_USER, '--force') + + # merge all diffs and push it into README.md or any other place + # notification def remote_status(self): From c624c9206cb27d7fc1eea878dd3d87ad9ddf4a0c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Oct 2018 13:56:42 +0300 Subject: [PATCH 05/84] Move UI into dashboard --- cvat/apps/git/__init__.py | 1 - .../apps/git/static/git/js/dashboardPlugin.js | 275 ++++++++++++++++-- cvat/apps/git/static/git/js/enginePlugin.js | 212 -------------- cvat/apps/git/urls.py | 4 +- cvat/apps/git/views.py | 33 ++- 5 files changed, 270 insertions(+), 255 deletions(-) delete mode 100644 cvat/apps/git/static/git/js/enginePlugin.js diff --git a/cvat/apps/git/__init__.py b/cvat/apps/git/__init__.py index 5c84ffca484b..a7027503b5db 100644 --- a/cvat/apps/git/__init__.py +++ b/cvat/apps/git/__init__.py @@ -5,4 +5,3 @@ from cvat.settings.base import JS_3RDPARTY JS_3RDPARTY['dashboard'] = JS_3RDPARTY.get('dashboard', []) + ['git/js/dashboardPlugin.js'] -JS_3RDPARTY['engine'] = JS_3RDPARTY.get('engine', []) + ['git/js/enginePlugin.js'] diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 035c5ca56345..5ec7e19f5512 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -6,41 +6,264 @@ "use strict"; +window.cvat = window.cvat || {}; +window.cvat.dashboard = window.cvat.dashboard || {}; +window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || []; +window.cvat.dashboard.uiCallbacks.push(function(newElements) { + let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); + + newElements.each(function(idx) { + let elem = $(newElements[idx]); + let tid = +elem.attr('id').split('_')[1]; + + $('').addClass('semiBold dashboardButtonUI') + .on('click', () => { + gitDialogWindow.attr('current_tid', tid); + gitDialogWindow.removeClass('hidden'); + window.cvat.git.updateState(); + }).appendTo(elem.find('div.dashboardButtonsUI')[0]); + }); +}); + +window.cvat.git = { + reposWindowId: 'gitReposWindow', + closeReposWindowButtonId: 'closeGitReposButton', + reposURLInputTextId: 'gitReposInputText', + reposURLUpdateButtonId: 'gitReposUpdateButton', + reposPushButtonId: 'gitReposPushButton', + labelStatusId: 'gitReposLabelStatus', + labelMessageId: 'gitReposLabelMessage', + createURLInputTextId: 'gitCreateURLInputText', + + updateState: () => { + /* Used unicode characters: + updating: ⏲ + pushed: ✓ + actual: ★ + obsolete: ☆ + error: ⚠ + */ + + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`); + let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`); + let reposURLInput = $(`#${window.cvat.git.reposURLInputTextId}`); + + window.cvat.git.getGitURL((data) => { + if (!data.url.value) { + gitLabelMessage.css('color', 'black').text('Repository is not attached'); + reposURLInput.prop('value', 'Repository is not attached'); + return; + } + + reposURLInput.prop('value', data.url.value); + + if (!data.status.value) { + gitLabelStatus.css('color', 'red').text('\u26a0'); + gitLabelMessage.css('color', 'red').text(data.status.error); + return; + } + + if (data.status.value == "actual") { + gitLabelStatus.css('color', 'darkgreen').text('\u2605'); + gitLabelMessage.css('color', 'darkgreen').text('Repository contains actual data'); + } + else if (data.status.value == "obsolete") { + gitLabelStatus.css('color', 'darkgreen').text('\u2606'); + gitLabelMessage.css('color', 'black').text('Repository contains obsolete data'); + } + else { + let message = `Got unknown repository status: ${data.status.value}`; + gitLabelStatus.css('color', 'red').text('\u26a0'); + gitLabelMessage.css('color', 'red').text(message); + throw Error(message); + } + }, (data) => { + gitWindow.addClass('hidden'); + let message = `Error was occured during get an repos URL. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + }); + }, + + getGitURL: (success, error) => { + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + $.ajax({ + url: '/git/repository/get/' + gitWindow.attr('current_tid'), + type: 'GET', + success: success, + error: error + }); + }, + + removeGitURL: () => { + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + $.ajax({ + url: '/git/repository/delete/' + gitWindow.attr('current_tid'), + type: 'DELETE', + success: window.cvat.git.updateState, + error: (data) => { + let message = `Error was occured during deleting an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + }, + + updateGitURL: (url) => { + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + $.ajax({ + url: '/git/repository/update', + type: 'POST', + data: JSON.stringify({ + 'tid': +gitWindow.attr('current_tid'), + 'url': url, + }), + success: window.cvat.git.updateState, + error: (data) => { + let message = `Error was occured during updating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + }, + + createGitURL: (url) => { + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + $.ajax({ + url: '/git/repository/create', + type: 'POST', + data: JSON.stringify({ + 'tid': +gitWindow.attr('current_tid'), + 'url': url, + }), + success: window.cvat.git.updateState, + error: (data) => { + let message = `Error was occured during creating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + }); + } +} + + document.addEventListener("DOMContentLoaded", () => { + /* CREATE TASK PLUGIN PART */ $(` - + `).insertAfter($("#dashboardBugTrackerInput").parent().parent()); - let originalCreateTaskRequest = createTaskRequest; - + let originalCreateTaskRequest = window.createTaskRequest; window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) { - let originalOnSuccessCreate = onSuccessCreate; - - onSuccessCreate = (tid) => { - let gitURL = $('#gitAnnoReposInput').prop('value'); - - if (gitURL.length) { - $.ajax({ - type: 'POST', - url: '/git/repository/create', - data: JSON.stringify({ - 'tid': tid, - 'url': gitURL, - }), - contentType: 'application/json;charset=utf-8', - error: (data) => { - throw Error(`Warning: Can't create git record for task ${tid}: ${data.responseText}`); - }, - complete: () => { - originalOnSuccessCreate(); - } - }); + try { + let originalOnSuccessCreate = onSuccessCreate; + onSuccessCreate = (tid) => { + let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value'); + + if (gitURL.length) { + $.ajax({ + type: 'POST', + url: '/git/repository/create', + data: JSON.stringify({ + 'tid': tid, + 'url': gitURL, + }), + contentType: 'application/json;charset=utf-8', + error: (data) => { + throw Error(`Warning: Can't create git record for task ${tid}. ` + + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); + }, + complete: () => { + originalOnSuccessCreate(); + } + }); + } } } - - originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); + finally { + originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); + } } + + /* GIT MODAL WINDOW PLUGIN PART */ + $(``).appendTo('body'); + + let gitWindow = $(`#${window.cvat.git.reposWindowId}`); + let closeRepositoryWindowButton = $(`#${window.cvat.git.closeReposWindowButtonId}`); + let repositoryURLInput = $(`#${window.cvat.git.reposURLInputTextId}`); + let repositoryUpdateButton = $(`#${window.cvat.git.reposURLUpdateButtonId}`); + let repositoryPushButton = $(`#${window.cvat.git.reposPushButtonId}`); + + closeRepositoryWindowButton.on('click', () => { + gitWindow.addClass('hidden'); + }); + + repositoryUpdateButton.on('click', () => { + let gitURL = repositoryURLInput.prop('value').replace(/\s/g,''); + if (!gitURL) { + window.cvat.git.removeGitURL(); + } + else { + window.cvat.git.getGitURL((data) => { + if (!data.url.value) { + window.cvat.git.createGitURL(gitURL); + } + else { + window.cvat.git.updateGitURL(gitURL); + } + }, () => { + let message = `Error was occured during getting an git URL. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + }); + } + }); + + repositoryPushButton.on('click', () => { + // to do + }); }); diff --git a/cvat/apps/git/static/git/js/enginePlugin.js b/cvat/apps/git/static/git/js/enginePlugin.js deleted file mode 100644 index 84d132ca521d..000000000000 --- a/cvat/apps/git/static/git/js/enginePlugin.js +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (C) 2018 Intel Corporation - * - * SPDX-License-Identifier: MIT - */ - -"use strict"; - -document.addEventListener("DOMContentLoaded", () => { - let gitReposWindowID = 'gitReposWindow'; - let closeReposWindowButtonId = 'closeGitReposButton'; - let gitRepositoryURLInputTextId = 'gitReposInputText'; - let gitRepositoryURLUpdateButtonId = 'gitReposUpdateButton'; - let gitRepositoryPushButtonId = 'gitReposPushButton'; - let gitLabelStatusId = 'gitReposLabelStatus'; - let gitLabelMessageId = 'gitReposLabelMessage'; - - $(` - `).appendTo('#taskAnnotationCenterPanel'); - - let gitWindow = $(`#${gitReposWindowID}`); - - $(``).on('click', () => { - $('#annotationMenu').addClass('hidden'); - gitWindow.removeClass('hidden'); - updateRepositoryInfo(); - }).prependTo($('#engineMenuButtons')); - - /* Used unicode characters: - updating: ⏲ - pushed: ✓ - actual: ★ - obsolete: ☆ - error: ⚠ - */ - - let closeRepositoryWindowButton = $(`#${closeReposWindowButtonId}`); - let repositoryURLInput = $(`#${gitRepositoryURLInputTextId}`); - let gitLabelMessage = $(`#${gitLabelMessageId}`); - let gitLabelStatus = $(`#${gitLabelStatusId}`); - let repositoryUpdateButton = $(`#${gitRepositoryURLUpdateButtonId}`); - let repositoryPushButton = $(`#${gitRepositoryPushButtonId}`); - - repositoryURLInput.on('keyup keydown keypress', (e) => e.stopPropagation()); - - closeRepositoryWindowButton.on('click', () => { - gitWindow.addClass('hidden'); - }); - - repositoryUpdateButton.on('click', () => { - let gitURL = repositoryURLInput.prop('value').replace(/\s/g,''); - if (!gitURL) { - removeGitURL(); - } - else { - getGitURL((data) => { - if (!data.url.value) { - createGitURL(gitURL); - } - else { - updateGitURL(gitURL); - } - }, () => { - let message = `Get git URL errors. Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - }); - } - }); - - repositoryPushButton.on('click', () => { - // to do - }); - - function createGitURL(url) { - $.ajax({ - url: '/git/repository/create', - type: 'POST', - data: JSON.stringify({ - 'tid': window.cvat.task.id, - 'url': url, - }), - success: () => { - updateRepositoryInfo(); - }, - error: (data) => { - let message = `Can not update repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - }); - } - - function updateGitURL(url) { - $.ajax({ - url: '/git/repository/update', - type: 'POST', - data: JSON.stringify({ - 'jid': window.cvat.job.id, - 'url': url, - }), - success: () => { - updateRepositoryInfo(); - }, - error: (data) => { - let message = `Can not update repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - }); - } - - function removeGitURL() { - $.ajax({ - url: '/git/repository/delete/' + window.cvat.job.id, - type: 'DELETE', - success: () => { - updateRepositoryInfo(); - }, - error: (data) => { - let message = `Can not delete repository entry. Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - }); - } - - function getGitURL(success, error) { - $.ajax({ - url: '/git/repository/get/' + window.cvat.job.id, - type: 'GET', - success: success, - error: error - }); - } - - function updateRepositoryInfo() { - gitLabelMessage.css('color', 'yellowgreen').text('Updating..'); - gitLabelStatus.css('color', 'yellowgreen').text('\u23F2'); - - getGitURL((data) => { - if (!data.url.value) { - gitLabelMessage.css('color', 'black').text('Repository is not attached'); - repositoryURLInput.prop('value', 'Repository is not attached'); - return; - } - - repositoryURLInput.prop('value', data.url.value); - - if (!data.status.value) { - gitLabelStatus.css('color', 'red').text('\u26a0'); - gitLabelMessage.css('color', 'red').text(data.status.error); - return; - } - - if (data.status.value == "actual") { - gitLabelStatus.css('color', 'darkgreen').text('\u2605'); - gitLabelMessage.css('color', 'darkgreen').text('Repository contains actual data'); - } - else if (data.status.value == "obsolete") { - gitLabelStatus.css('color', 'darkgreen').text('\u2606'); - gitLabelMessage.css('color', 'black').text('Repository contains obsolete data'); - } - else { - let message = `Got unknown repository status: ${data.status.value}`; - gitLabelStatus.css('color', 'red').text('\u26a0'); - gitLabelMessage.css('color', 'red').text(message); - throw Error(message); - } - }, (data) => { - gitWindow.addClass('hidden'); - let message = `Get respository URL errors. Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - }); - } -}); diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py index 5e816c6fbf36..42cb03d6c447 100644 --- a/cvat/apps/git/urls.py +++ b/cvat/apps/git/urls.py @@ -10,6 +10,6 @@ urlpatterns = [ path('create', views.create_repository), path('update', views.update_repository), - path('get/', views.get_repository), - path('delete/', views.delete_repository), + path('get/', views.get_repository), + path('delete/', views.delete_repository), ] \ No newline at end of file diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 25bb04804892..b005b36b8a55 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -5,11 +5,14 @@ from django.shortcuts import render from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.contrib.auth.decorators import permission_required +from django.db import transaction from cvat.apps.authentication.decorators import login_required from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Task, Job from cvat.apps.git.models import GitData +from cvat.apps.git.git import Git + import json @@ -37,22 +40,23 @@ def create_repository(request): return HttpResponse() +@transaction.atomic @login_required @permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) def update_repository(request): try: data = json.loads(request.body.decode('utf-8')) - jid = data['jid'] + tid = data['tid'] url = data['url'] - db_job = Job.objects.select_related('segment__task').get(pk = jid) + db_task = Task.objects.get(pk = tid) - db_git = GitData.objects.select_for_update().get(pk = db_job.segment.task) + db_git = GitData.objects.select_for_update().get(pk = db_task) db_git.url = url db_git.save() except Exception as e: try: - slogger.job[jid].error("can not update git repository", exc_info=True) + slogger.task[tid].error("can not update git repository", exc_info=True) except: pass @@ -63,7 +67,7 @@ def update_repository(request): @login_required @permission_required(perm=['engine.view_task'], raise_exception=True) -def get_repository(request, jid): +def get_repository(request, tid): try: response = { 'url': { @@ -75,35 +79,36 @@ def get_repository(request, jid): } } - db_job = Job.objects.select_related("segment__task").get(pk = jid) - if not GitData.objects.filter(pk = db_job.segment.task).exists(): + db_task = Task.objects.get(pk = tid) + if not GitData.objects.filter(pk = db_task).exists(): return JsonResponse(response) - response['url']['value'] = GitData.objects.get(pk = db_job.segment.task).url + response['url']['value'] = GitData.objects.get(pk = db_task).url response['status']['error'] = 'not implemented' return JsonResponse(response) except Exception as e: try: - slogger.job[jid].error("can not get git repository info", exc_info=True) + slogger.task[tid].error("can not get git repository info", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) +@transaction.atomic @login_required @permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) -def delete_repository(request, jid): +def delete_repository(request, tid): try: - db_job = Job.objects.select_related("segment__task").get(pk = jid) + db_task = Task.objects.get(pk = tid) - if GitData.objects.filter(pk = db_job.segment.task).exists(): - db_git = GitData.objects.select_for_update().get(pk = db_job.segment.task) + if GitData.objects.filter(pk = db_task).exists(): + db_git = GitData.objects.select_for_update().get(pk = db_task) db_git.delete() except Exception as e: try: - slogger.job[jid].error("can not delete git repository data", exc_info=True) + slogger.task[tid].error("can not delete git repository data", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) From 5f6f582ac00cdb13bbb2903084543141afa49f54 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Oct 2018 14:00:31 +0300 Subject: [PATCH 06/84] Couple of fixes --- cvat/apps/dashboard/static/dashboard/js/dashboard.js | 7 +++---- cvat/apps/engine/static/engine/js/annotationUI.js | 3 --- cvat/apps/git/git.py | 2 -- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/cvat/apps/dashboard/static/dashboard/js/dashboard.js b/cvat/apps/dashboard/static/dashboard/js/dashboard.js index 4dbd60cbcd33..6e95eee5ed3f 100644 --- a/cvat/apps/dashboard/static/dashboard/js/dashboard.js +++ b/cvat/apps/dashboard/static/dashboard/js/dashboard.js @@ -68,10 +68,9 @@ function buildDashboard() { setupTaskUpdater(); setupSearch(); - $(window).on('click', function(e) { - let target = $(e.target); - if ( target.hasClass('modal') ) { - target.addClass('hidden'); + $(window).on('click', function(event) { + if (event.target.classList.contains('modal')) { + event.target.classList.add('hidden'); } }); diff --git a/cvat/apps/engine/static/engine/js/annotationUI.js b/cvat/apps/engine/static/engine/js/annotationUI.js index db757bc7b75b..43c54e655df3 100644 --- a/cvat/apps/engine/static/engine/js/annotationUI.js +++ b/cvat/apps/engine/static/engine/js/annotationUI.js @@ -57,9 +57,6 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) { id: job.jobid, images: job.image_meta_data, }, - task: { - id: job.taskid - }, search: { value: window.location.search, diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index aede86ad7cae..aa8b395e3a01 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: MIT -#!/bin/bash - import datetime import shutil import json From 2794eb1b6c8f3441c1b82e2f19a6ffc754cf0799 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Oct 2018 14:09:34 +0300 Subject: [PATCH 07/84] Again some fixes --- cvat/apps/git/static/git/js/dashboardPlugin.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 5ec7e19f5512..dbe371caa1c7 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -10,14 +10,13 @@ window.cvat = window.cvat || {}; window.cvat.dashboard = window.cvat.dashboard || {}; window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || []; window.cvat.dashboard.uiCallbacks.push(function(newElements) { - let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); - newElements.each(function(idx) { let elem = $(newElements[idx]); let tid = +elem.attr('id').split('_')[1]; $('').addClass('semiBold dashboardButtonUI') .on('click', () => { + let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); gitDialogWindow.attr('current_tid', tid); gitDialogWindow.removeClass('hidden'); window.cvat.git.updateState(); @@ -52,10 +51,11 @@ window.cvat.git = { window.cvat.git.getGitURL((data) => { if (!data.url.value) { gitLabelMessage.css('color', 'black').text('Repository is not attached'); - reposURLInput.prop('value', 'Repository is not attached'); + reposURLInput.attr('placeholder', 'Repository is not attached'); return; } + reposURLInput.attr('placeholder', ''); reposURLInput.prop('value', data.url.value); if (!data.status.value) { From 0a59aed85b01005ae683364395b9807188bbb633 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Oct 2018 14:19:28 +0300 Subject: [PATCH 08/84] Some appearance changes --- .../apps/dashboard/static/dashboard/stylesheet.css | 14 +++++++------- cvat/apps/dashboard/templates/dashboard/task.html | 10 +++++----- cvat/apps/git/static/git/js/dashboardPlugin.js | 2 +- .../static/tf_annotation/js/tf_annotation.js | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cvat/apps/dashboard/static/dashboard/stylesheet.css b/cvat/apps/dashboard/static/dashboard/stylesheet.css index a3430e857561..a5a047fc2f6f 100644 --- a/cvat/apps/dashboard/static/dashboard/stylesheet.css +++ b/cvat/apps/dashboard/static/dashboard/stylesheet.css @@ -14,7 +14,7 @@ } .dashboardTaskIntro { - width: 25%; + width: 30%; height: 75%; float: left; margin-left: 20px; @@ -25,7 +25,7 @@ .dashboardButtonsUI { margin-top: 1%; - width: 33%; + width: 35%; height: 75%; float: left; overflow-y: auto; @@ -33,19 +33,19 @@ .dashboardButtonUI { display: block; - width: 70%; - height: 2.5em; + width: 60%; + height: 1.6em; margin: auto; - margin-top: 0.1em; + margin-top: 0.3em; font-size: 1em; } .dashboardJobsUI { - width: 40%; + width: 30%; height: 75%; float: left; text-align: center; - overflow-y: scroll; + overflow-y: auto; } .dashboardJobList { diff --git a/cvat/apps/dashboard/templates/dashboard/task.html b/cvat/apps/dashboard/templates/dashboard/task.html index da017006b59c..073fca8fadba 100644 --- a/cvat/apps/dashboard/templates/dashboard/task.html +++ b/cvat/apps/dashboard/templates/dashboard/task.html @@ -12,12 +12,12 @@
- - - - + + + + {%if item.bug_tracker %} - + {% endif %}
diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index dbe371caa1c7..eff523b49bab 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -14,7 +14,7 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) { let elem = $(newElements[idx]); let tid = +elem.attr('id').split('_')[1]; - $('').addClass('semiBold dashboardButtonUI') + $('').addClass('regular dashboardButtonUI') .on('click', () => { let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); gitDialogWindow.attr('current_tid', tid); diff --git a/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js b/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js index f7e0f3f0a03c..4deed70a88d7 100644 --- a/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js +++ b/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js @@ -110,7 +110,7 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) { let tfAnnotationButton = $(''); tfAnnotationButton.on('click', onTFAnnotationClick.bind(tfAnnotationButton)); - tfAnnotationButton.addClass('dashboardTFAnnotationButton semiBold dashboardButtonUI'); + tfAnnotationButton.addClass('dashboardTFAnnotationButton regular dashboardButtonUI'); tfAnnotationButton.appendTo(buttonsUI); if ((tid in data) && (data[tid].active)) { From 1c21e8aebd11daca69d7310340d904b0377eb113 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 15:56:02 +0300 Subject: [PATCH 09/84] Some fixes --- cvat/apps/git/git.py | 146 +++++++++++------- .../apps/git/static/git/js/dashboardPlugin.js | 38 ++++- cvat/apps/git/views.py | 31 ++-- 3 files changed, 146 insertions(+), 69 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index aa8b395e3a01..36394f530088 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +from cvat.apps.engine.log import slogger + import datetime import shutil import json @@ -13,7 +15,6 @@ CVAT_USER = "cvat" CVAT_EMAIL = "cvat@example.com" - class Git: __url = None __tid = None @@ -28,14 +29,6 @@ def __init__(self, url, tid, user): self.__user = user self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repository") - try: - self.__rep = git.Repo(self.__cwd) - except git.exc.InvalidGitRepositoryError: - shutil.rmtree(self.__cwd, True) - self.reclone() - except git.exc.NoSuchPathError: - self.clone() - def _parse_url(self): http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)" @@ -45,7 +38,6 @@ def _parse_url(self): ssh_match = re.match(ssh_pattern, self.__url) if http_match: - print("{} is http URL".format(self.__url)) user = "git" scheme = http_match.group(1) if http_match.group(1) else "https" host = http_match.group(2) @@ -54,7 +46,6 @@ def _parse_url(self): repos += ".git" return scheme, user, host, repos elif ssh_match: - print("{} is ssh URL".format(self.__url)) scheme = "https" user = ssh_match.group(1) host = ssh_match.group(2) @@ -65,14 +56,32 @@ def _parse_url(self): else: raise Exception("Couldn't parse URL") + def _create_master(self): - if not len(self.__rep.heads): - readme_md_name = os.path.join(self.__cwd, "README.md") - with open(readme_md_name, "w"): - pass - self.__rep.index.add([readme_md_name]) - self.__rep.head.reference = self.__rep.create_head("master") - self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) + assert not len(self.__rep.heads) + readme_md_name = os.path.join(self.__cwd, "README.md") + with open(readme_md_name, "w"): + pass + self.__rep.index.add([readme_md_name]) + self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user.username, datetime.datetime.now())) + + + def init_repos(self): + try: + # Try to use a local repos. It can throw GitError exception + self.__rep = git.Repo(self.__cwd) + + # Check if remote URL is actual + if self.ssh_url() != self.__rep.git.remote('get-url', '--all', 'origin'): + slogger.task[self.__tid].info("Local repository URL is obsolete.") + # We need reinitialize repository if it's false + raise git.exc.GitError + except git.exc.GitError: + slogger.task[self.__tid].info("Local repository reinitialization..") + shutil.rmtree(self.__cwd, True) + self.clone() + + return self def ssh_url(self): @@ -90,8 +99,13 @@ def http_url(self): def clone(self): os.makedirs(self.__cwd) ssh_url = self.ssh_url() + + # Clone repository + slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url)) self.__rep = git.Repo.clone_from(ssh_url, self.__cwd) + # Setup config file for CVAT user + slogger.task[self.__tid].info("User config initialization..") with self.__rep.config_writer() as cw: if not cw.has_section("user"): cw.add_section("user") @@ -99,12 +113,16 @@ def clone(self): cw.set("user", "email", CVAT_EMAIL) cw.release() - self._create_master() - try: - self.__rep.git.push("origin", "master") - except git.exc.GitError: - print("Remote master branch wasn't found, but script couldn't push it") + # Create master branch if it doesn't exist and push it into server for remote repository initialization + if not len(self.__rep.heads): + self._create_master() + try: + self.__rep.git.push("origin", "master") + except git.exc.GitError: + slogger.task[self.__tid].warning("Remote repository doesn't contain any " + + "heads but master push process is fault", exc_info = True) + # Create dir for diffs and add it into .gitignore os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) gitignore = os.path.join(self.__cwd, ".gitignore") file_mode = "a" if os.path.isfile(os.path.join(self.__cwd, ".gitignore")) else "w" @@ -112,26 +130,32 @@ def clone(self): gitignore.writelines(["\n", "{}/".format(DIFF_DIR), "\n"]) + # Reclone method instead of clone try to save all accumulated diffs def reclone(self): if os.path.exists(self.__cwd): if not os.path.isdir(self.__cwd): os.remove(self.__cwd) else: - tmp_repo = os.path.join(os.path.split(self.__cwd)[:-1], "tmp_repo".format(self.__user)) + # Rename current repository dir + tmp_repo = os.path.join(self.__cwd, "..", "tmp_repo") os.rename(self.__cwd, tmp_repo) successful_cloning = False try: self.clone() successful_cloning = True - except git.exc.GitError: + except Exception as ex: + # Restore state if any error has been occured + if os.path.isdir(self.__cwd): + shutil.rmtree(self.__cwd, True) os.rename(tmp_repo, self.__cwd) - raise Exception("Couldn't clone repository") + raise ex + # If clone process is successful, push all diffs into new local repository if successful_cloning: if os.path.exists(os.path.join(tmp_repo, DIFF_DIR)): diffs_to_move = list(map(lambda x: os.path.join(DIFF_DIR, x), os.listdir(os.path.join(tmp_repo, DIFF_DIR)))) - diffs_to_move = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff"), diffs_to_move) + diffs_to_move = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs_to_move)) for diff in diffs_to_move: os.rename(os.path.join(tmp_repo, diff), os.path.join(self.__cwd, diff)) @@ -143,54 +167,72 @@ def reclone(self): def onsave(self, changes_dict): diff_dir = os.path.join(self.__cwd, DIFF_DIR) - os.makedirs(diff_dir, exist_ok=True) - diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - last_num = 0 - for f in diff_files: - number = f.split("_")[0] - number = int(number) if number.isdigit() else last_num - last_num = number - - with open("{}_{}.diff".format(last_num + 1, self.__user), 'w') as f: - f.write(json.dumps(changes_dict)) + if os.path.isdir(diff_dir): + diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + last_num = 0 + for f in diff_files: + number = f.split("_")[0] + number = int(number) if number.isdigit() else last_num + last_num = number + + with open("{}_{}.diff".format(last_num + 1, self.__user.username), 'w') as f: + f.write(json.dumps(changes_dict)) + else: + raise Exception("Local repository isn't found") def pull(self): - if "master" not in self.__rep.heads: - self._create_master() - + self.__rep.head.reference = self.__rep.heads["master"] remote_branches = [] for remote_branch in self.__rep.git.branch("-r").split("\n")[1:]: remote_branches.append(remote_branch.split("/")[-1]) - if "master" in remote_branches: try: self.__rep.git.pull("origin", "master") except git.exc.GitError: + # Merge conflicts self.reclone() def push(self): + # Update local repository self.pull() - if CVAT_USER in list(map(lambda x: x.name, self.__rep.heads)): - git.Head.delete(self.__rep.heads[CVAT_USER].repo, self.__rep.heads[CVAT_USER]) - self.__rep.create_head(CVAT_USER) + # Remove user branch from local repository if it exists + if self.__user.username in list(map(lambda x: x.name, self.__rep.heads)): + git.Head.delete(self.__rep.heads[self.__user.username].repo, self.__rep.heads[self.__user.username]) + + # Create new user branch from master + self.__rep.create_head(self.__user.username) + + # Dump annotation and zip it os.makedirs(os.path.join(self.__cwd, 'annotation'), exist_ok = True) with open(os.path.join(self.__cwd, 'annotation', 'SomeTask.dump'), 'w'): - # create fake dump file pass + # Add to index self.__rep.index.add([os.path.join(self.__cwd, 'annotation', '*.dump')]) - self.__rep.git.push("origin", CVAT_USER, '--force') + self.__rep.git.push("origin", self.__user.username, '--force') + - # merge all diffs and push it into README.md or any other place - # notification + # TODO: + # 1) Checkout from other branches (not only master) + # 2) Dump real file + # 3) ZIP real file + # 4) Merge diffs into one file with name summary.diffs + # This file contains diffs by date + # Notification + + def delete(self): + if os.path.isdir(self.__cwd): + shutil.rmtree(self.__cwd) def remote_status(self): diff_dir = os.path.join(self.__cwd, DIFF_DIR) - os.makedirs(diff_dir, exist_ok=True) - diffs = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs)) - return "actual" if not len(diffs) else "obsolete" + if os.path.isdir(diff_dir): + diffs = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs)) + return "actual" if not len(diffs) else "obsolete" + else: + raise Exception("Local repository isn't found") diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index eff523b49bab..6151c1086128 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -47,8 +47,17 @@ window.cvat.git = { let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`); let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`); let reposURLInput = $(`#${window.cvat.git.reposURLInputTextId}`); + let updateButton = $(`#${window.cvat.git.reposURLUpdateButtonId}`); + let pushButton = $(`#${window.cvat.git.reposPushButtonId}`); + + gitLabelMessage.css('color', '#cccc00').text('Getting an info..'); + gitLabelStatus.css('color', '#cccc00').text('\u25cc'); + updateButton.attr("disabled", true); + pushButton.attr("disabled", true); window.cvat.git.getGitURL((data) => { + updateButton.attr("disabled", false); + if (!data.url.value) { gitLabelMessage.css('color', 'black').text('Repository is not attached'); reposURLInput.attr('placeholder', 'Repository is not attached'); @@ -71,6 +80,7 @@ window.cvat.git = { else if (data.status.value == "obsolete") { gitLabelStatus.css('color', 'darkgreen').text('\u2606'); gitLabelMessage.css('color', 'black').text('Repository contains obsolete data'); + pushButton.attr("disabled", false); } else { let message = `Got unknown repository status: ${data.status.value}`; @@ -79,6 +89,7 @@ window.cvat.git = { throw Error(message); } }, (data) => { + updateButton.attr("disabled", false); gitWindow.addClass('hidden'); let message = `Error was occured during get an repos URL. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; @@ -166,7 +177,7 @@ document.addEventListener("DOMContentLoaded", () => { try { let originalOnSuccessCreate = onSuccessCreate; onSuccessCreate = (tid) => { - let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value'); + let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value').replace(/\s/g,''); if (gitURL.length) { $.ajax({ @@ -178,7 +189,7 @@ document.addEventListener("DOMContentLoaded", () => { }), contentType: 'application/json;charset=utf-8', error: (data) => { - throw Error(`Warning: Can't create git record for task ${tid}. ` + + console.log(`Warning: Can't create git record for task ${tid}. ` + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); }, complete: () => { @@ -186,6 +197,9 @@ document.addEventListener("DOMContentLoaded", () => { } }); } + else { + originalOnSuccessCreate(); + } } } finally { @@ -236,23 +250,35 @@ document.addEventListener("DOMContentLoaded", () => { let repositoryURLInput = $(`#${window.cvat.git.reposURLInputTextId}`); let repositoryUpdateButton = $(`#${window.cvat.git.reposURLUpdateButtonId}`); let repositoryPushButton = $(`#${window.cvat.git.reposPushButtonId}`); + let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`); + let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`); closeRepositoryWindowButton.on('click', () => { gitWindow.addClass('hidden'); }); repositoryUpdateButton.on('click', () => { + gitLabelMessage.css('color', '#cccc00').text('Updating..'); + gitLabelStatus.css('color', '#cccc00').text('\u25cc'); + repositoryUpdateButton.attr("disabled", true); + repositoryPushButton.attr("disabled", true); + let gitURL = repositoryURLInput.prop('value').replace(/\s/g,''); if (!gitURL) { window.cvat.git.removeGitURL(); } else { window.cvat.git.getGitURL((data) => { - if (!data.url.value) { - window.cvat.git.createGitURL(gitURL); + try { + if (!data.url.value) { + window.cvat.git.createGitURL(gitURL); + } + else { + window.cvat.git.updateGitURL(gitURL); + } } - else { - window.cvat.git.updateGitURL(gitURL); + finally { + window.cvat.git.updateState(); } }, () => { let message = `Error was occured during getting an git URL. ` + diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index b005b36b8a55..c784f57a7d06 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -13,10 +13,11 @@ from cvat.apps.git.models import GitData from cvat.apps.git.git import Git - import json +import git +@transaction.atomic @login_required @permission_required('engine.add_task', raise_exception=True) def create_repository(request): @@ -30,13 +31,17 @@ def create_repository(request): raise Exception('git repository for task already exists') db_git = GitData() - db_git.task = db_task db_git.url = url + db_git.task = db_task db_git.save() + + db_git = GitData.objects.select_for_update().get(pk = db_task) + Git(url, tid, request.user).init_repos() except Exception as e: + if isinstance(db_git, GitData): + db_git.delete() slogger.glob.error("cannot create git repository for task #{}".format(tid), exc_info=True) return HttpResponseBadRequest(str(e)) - return HttpResponse() @@ -49,9 +54,8 @@ def update_repository(request): tid = data['tid'] url = data['url'] - db_task = Task.objects.get(pk = tid) - - db_git = GitData.objects.select_for_update().get(pk = db_task) + db_git = GitData.objects.select_for_update().get(pk = Task.objects.get(pk = tid)) + Git(url, tid, request.user).init_repos() db_git.url = url db_git.save() except Exception as e: @@ -59,12 +63,11 @@ def update_repository(request): slogger.task[tid].error("can not update git repository", exc_info=True) except: pass - return HttpResponseBadRequest(str(e)) - return HttpResponse() +@transaction.atomic @login_required @permission_required(perm=['engine.view_task'], raise_exception=True) def get_repository(request, tid): @@ -83,10 +86,13 @@ def get_repository(request, tid): if not GitData.objects.filter(pk = db_task).exists(): return JsonResponse(response) - response['url']['value'] = GitData.objects.get(pk = db_task).url - response['status']['error'] = 'not implemented' + db_git = GitData.objects.select_for_update().get(pk = db_task) - return JsonResponse(response) + response['url']['value'] = db_git.url + try: + response['status']['value'] = Git(db_git.url, tid, request.user).remote_status() + except Exception as ex: + response['status']['error'] = str(ex) except Exception as e: try: slogger.task[tid].error("can not get git repository info", exc_info=True) @@ -94,6 +100,8 @@ def get_repository(request, tid): pass return HttpResponseBadRequest(str(e)) + return JsonResponse(response) + @transaction.atomic @login_required @@ -104,6 +112,7 @@ def delete_repository(request, tid): if GitData.objects.filter(pk = db_task).exists(): db_git = GitData.objects.select_for_update().get(pk = db_task) + Git(db_git.url, tid, request.user).delete() db_git.delete() except Exception as e: From a7d21ba7136616de14c36720567e201a077a1011 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 16:07:15 +0300 Subject: [PATCH 10/84] Fixed updating --- cvat/apps/git/static/git/js/dashboardPlugin.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 6151c1086128..248a5398fb73 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -132,7 +132,7 @@ window.cvat.git = { 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - success: window.cvat.git.updateState, + complete: window.cvat.git.updateState, error: (data) => { let message = `Error was occured during updating an repos entry. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; @@ -151,7 +151,7 @@ window.cvat.git = { 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - success: window.cvat.git.updateState, + complete: window.cvat.git.updateState, error: (data) => { let message = `Error was occured during creating an repos entry. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; @@ -269,16 +269,11 @@ document.addEventListener("DOMContentLoaded", () => { } else { window.cvat.git.getGitURL((data) => { - try { - if (!data.url.value) { - window.cvat.git.createGitURL(gitURL); - } - else { - window.cvat.git.updateGitURL(gitURL); - } + if (!data.url.value) { + window.cvat.git.createGitURL(gitURL); } - finally { - window.cvat.git.updateState(); + else { + window.cvat.git.updateGitURL(gitURL); } }, () => { let message = `Error was occured during getting an git URL. ` + From 8c6db07c8e225afc26e8b2726d5c4aeed2bf2b0a Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 16:11:02 +0300 Subject: [PATCH 11/84] Again fix --- .../apps/git/static/git/js/dashboardPlugin.js | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 248a5398fb73..fde06da62dbe 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -132,12 +132,17 @@ window.cvat.git = { 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - complete: window.cvat.git.updateState, + success: window.cvat.git.updateState, error: (data) => { - let message = `Error was occured during updating an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); + try { + let message = `Error was occured during updating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + finally { + window.cvat.git.updateState(); + } } }); }, @@ -151,12 +156,17 @@ window.cvat.git = { 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - complete: window.cvat.git.updateState, + success: window.cvat.git.updateState, error: (data) => { - let message = `Error was occured during creating an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); + try { + let message = `Error was occured during creating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + finally { + window.cvat.git.updateState(); + } } }); } From da698072e2c154bfe37337353035891b8e4fc6e7 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 17:00:03 +0300 Subject: [PATCH 12/84] Changed request structure --- cvat/apps/git/git.py | 86 +++++++++++++++++++++++++++++++++++++++--- cvat/apps/git/views.py | 74 ++++++++---------------------------- 2 files changed, 96 insertions(+), 64 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 36394f530088..05bac3918528 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -2,7 +2,11 @@ # # SPDX-License-Identifier: MIT +from django.db import transaction + from cvat.apps.engine.log import slogger +from cvat.apps.engine.models import Task +from cvat.apps.git.models import GitData import datetime import shutil @@ -15,6 +19,7 @@ CVAT_USER = "cvat" CVAT_EMAIL = "cvat@example.com" + class Git: __url = None __tid = None @@ -216,12 +221,20 @@ def push(self): # TODO: - # 1) Checkout from other branches (not only master) - # 2) Dump real file - # 3) ZIP real file - # 4) Merge diffs into one file with name summary.diffs - # This file contains diffs by date + # 1) Dump real file + # 2) ZIP real file + # 3) Merge diffs into one file with name summary.diff. This file contains diffs by date + # 4) LFS + # 5) Using workers # Notification + # + # + # Setup it in the container + # 1) Register CVAT user. Create SSH keys for it. + # 2) + # + # Future: + # 1) Checkout from other branches (not only master def delete(self): if os.path.isdir(self.__cwd): @@ -236,3 +249,66 @@ def remote_status(self): return "actual" if not len(diffs) else "obsolete" else: raise Exception("Local repository isn't found") + + +@transaction.atomic +def create(url, tid, user): + try: + db_task = Task.objects.get(pk = tid) + if GitData.objects.filter(pk = db_task).exists(): + raise Exception('git repository for task already exists') + + db_git = GitData() + db_git.url = url + db_git.task = db_task + db_git.save() + + db_git = GitData.objects.select_for_update().get(pk = db_task) + Git(url, tid, user).init_repos() + except Exception as ex: + if isinstance(db_git, GitData): + db_git.delete() + raise ex + + +@transaction.atomic +def update(url, tid, user): + db_task = Task.objects.get(pk = tid) + db_git = GitData.objects.select_for_update().get(pk = db_task) + + Git(url, tid, user).init_repos() + + db_git.url = url + db_git.save() + + +@transaction.atomic +def get(tid, user): + response = {} + response["url"] = {"value": None} + response["status"] = {"value": None, "error": None} + + db_task = Task.objects.get(pk = tid) + if GitData.objects.filter(pk = db_task).exists(): + db_git = GitData.objects.select_for_update().get(pk = db_task) + response['url']['value'] = db_git.url + try: + response['status']['value'] = Git(db_git.url, tid, user).remote_status() + except Exception as ex: + response['status']['error'] = str(ex) + return response + + +@transaction.atomic +def delete(tid, user): + db_task = Task.objects.get(pk = tid) + + if GitData.objects.filter(pk = db_task).exists(): + db_git = GitData.objects.select_for_update().get(pk = db_task) + Git(db_git.url, tid, user).delete() + db_git.delete() + + + + + diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index c784f57a7d06..3e922ee5d16b 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -2,122 +2,78 @@ # # SPDX-License-Identifier: MIT -from django.shortcuts import render from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.contrib.auth.decorators import permission_required from django.db import transaction from cvat.apps.authentication.decorators import login_required from cvat.apps.engine.log import slogger -from cvat.apps.engine.models import Task, Job -from cvat.apps.git.models import GitData -from cvat.apps.git.git import Git +import cvat.apps.git.git as CVATGit import json import git -@transaction.atomic @login_required @permission_required('engine.add_task', raise_exception=True) def create_repository(request): try: + tid = None data = json.loads(request.body.decode('utf-8')) tid = data['tid'] url = data['url'] - db_task = Task.objects.get(pk = tid) - if GitData.objects.filter(pk = db_task).exists(): - raise Exception('git repository for task already exists') - - db_git = GitData() - db_git.url = url - db_git.task = db_task - db_git.save() - - db_git = GitData.objects.select_for_update().get(pk = db_task) - Git(url, tid, request.user).init_repos() + slogger.glob.info("create repository request for task #{}".format(tid)) + CVATGit.create(url, tid, request.user) except Exception as e: - if isinstance(db_git, GitData): - db_git.delete() - slogger.glob.error("cannot create git repository for task #{}".format(tid), exc_info=True) + slogger.glob.error("error has been occured during deleting repository for the task #{}".format(tid), exc_info=True) return HttpResponseBadRequest(str(e)) return HttpResponse() -@transaction.atomic @login_required @permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) def update_repository(request): try: + tid = None data = json.loads(request.body.decode('utf-8')) tid = data['tid'] url = data['url'] - db_git = GitData.objects.select_for_update().get(pk = Task.objects.get(pk = tid)) - Git(url, tid, request.user).init_repos() - db_git.url = url - db_git.save() + slogger.glob.info("update repository request for task #{}".format(tid)) + CVATGit.update(url, tid, request.user) except Exception as e: try: - slogger.task[tid].error("can not update git repository", exc_info=True) + slogger.task[tid].error("error has been occured during updating a git repository", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) return HttpResponse() -@transaction.atomic @login_required @permission_required(perm=['engine.view_task'], raise_exception=True) def get_repository(request, tid): try: - response = { - 'url': { - 'value': None, - }, - 'status': { - 'value': None, - 'error': None - } - } - - db_task = Task.objects.get(pk = tid) - if not GitData.objects.filter(pk = db_task).exists(): - return JsonResponse(response) - - db_git = GitData.objects.select_for_update().get(pk = db_task) - - response['url']['value'] = db_git.url - try: - response['status']['value'] = Git(db_git.url, tid, request.user).remote_status() - except Exception as ex: - response['status']['error'] = str(ex) + slogger.glob.info("get repository request for task #{}".format(tid)) + return JsonResponse(CVATGit.get(tid, request.user)) except Exception as e: try: - slogger.task[tid].error("can not get git repository info", exc_info=True) + slogger.task[tid].error("error has been occured during getting repository info", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) - return JsonResponse(response) - -@transaction.atomic @login_required @permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) def delete_repository(request, tid): try: - db_task = Task.objects.get(pk = tid) - - if GitData.objects.filter(pk = db_task).exists(): - db_git = GitData.objects.select_for_update().get(pk = db_task) - Git(db_git.url, tid, request.user).delete() - db_git.delete() - + slogger.glob.info("delete repository request for task #{}".format(tid)) + CVATGit.delete(tid, request.user) except Exception as e: try: - slogger.task[tid].error("can not delete git repository data", exc_info=True) + slogger.task[tid].error("error has been occured during deleting a repository", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) From 1d02912c64c04d84c8c570258910a8f51f503deb Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 17:02:56 +0300 Subject: [PATCH 13/84] Removed extra comments --- cvat/apps/git/git.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 05bac3918528..d131ce854e21 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -220,22 +220,6 @@ def push(self): self.__rep.git.push("origin", self.__user.username, '--force') - # TODO: - # 1) Dump real file - # 2) ZIP real file - # 3) Merge diffs into one file with name summary.diff. This file contains diffs by date - # 4) LFS - # 5) Using workers - # Notification - # - # - # Setup it in the container - # 1) Register CVAT user. Create SSH keys for it. - # 2) - # - # Future: - # 1) Checkout from other branches (not only master - def delete(self): if os.path.isdir(self.__cwd): shutil.rmtree(self.__cwd) @@ -307,8 +291,3 @@ def delete(tid, user): db_git = GitData.objects.select_for_update().get(pk = db_task) Git(db_git.url, tid, user).delete() db_git.delete() - - - - - From 9c7c640329fc7e10610c0954aa55ee23f50f6704 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 17:47:38 +0300 Subject: [PATCH 14/84] Sync dump & push --- cvat/apps/git/git.py | 37 +++++++++++++++++++++++++++++-------- cvat/apps/git/urls.py | 1 + cvat/apps/git/views.py | 29 ++++++++++++++++++++++------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index d131ce854e21..a0b60a0d6129 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -6,8 +6,10 @@ from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Task +from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML from cvat.apps.git.models import GitData +import subprocess import datetime import shutil import json @@ -199,7 +201,7 @@ def pull(self): self.reclone() - def push(self): + def push(self, scheme, host, format): # Update local repository self.pull() @@ -210,13 +212,25 @@ def push(self): # Create new user branch from master self.__rep.create_head(self.__user.username) - # Dump annotation and zip it - os.makedirs(os.path.join(self.__cwd, 'annotation'), exist_ok = True) - with open(os.path.join(self.__cwd, 'annotation', 'SomeTask.dump'), 'w'): - pass - - # Add to index - self.__rep.index.add([os.path.join(self.__cwd, 'annotation', '*.dump')]) + # Dump and zip + dump(self.__tid, format, scheme, host) + db_task = Task.objects.get(pk = self.__tid) + dump_name = db_task.get_dump_path() + + if not os.path.isfile(dump_name): + raise Exception("Dump completed but file isn't found") + base_path = os.path.join(self.__cwd, "annotation") + shutil.rmtree(base_path, True) + os.makedirs(base_path, exist_ok = True) + new_dump_name = os.path.join(base_path, os.path.basename(dump_name)) + os.rename(dump_name, new_dump_name) + archive_name = os.path.join(base_path, "annotation.zip") + subprocess.call('zip -r "{}" "{}"'.format(archive_name, new_dump_name), shell=True) + os.remove(new_dump_name) + + # Commit and push + self.__rep.index.add([archive_name]) + self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user.username, datetime.datetime.now())) self.__rep.git.push("origin", self.__user.username, '--force') @@ -266,6 +280,13 @@ def update(url, tid, user): db_git.save() +@transaction.atomic +def push(tid, user, scheme, host): + db_task = Task.objects.get(pk = tid) + db_git = GitData.objects.select_for_update().get(pk = db_task) + Git(db_git.url, tid, user).init_repos().push(scheme, host, FORMAT_XML) + + @transaction.atomic def get(tid, user): response = {} diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py index 42cb03d6c447..24c3a08a8273 100644 --- a/cvat/apps/git/urls.py +++ b/cvat/apps/git/urls.py @@ -12,4 +12,5 @@ path('update', views.update_repository), path('get/', views.get_repository), path('delete/', views.delete_repository), + path('push/', views.push_repository), ] \ No newline at end of file diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 3e922ee5d16b..2bea3132bd6e 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -26,7 +26,7 @@ def create_repository(request): slogger.glob.info("create repository request for task #{}".format(tid)) CVATGit.create(url, tid, request.user) except Exception as e: - slogger.glob.error("error has been occured during deleting repository for the task #{}".format(tid), exc_info=True) + slogger.glob.error("error has been occured during creating repository request for the task #{}".format(tid), exc_info=True) return HttpResponseBadRequest(str(e)) return HttpResponse() @@ -40,11 +40,26 @@ def update_repository(request): tid = data['tid'] url = data['url'] - slogger.glob.info("update repository request for task #{}".format(tid)) + slogger.task[tid].info("update repository request") CVATGit.update(url, tid, request.user) except Exception as e: try: - slogger.task[tid].error("error has been occured during updating a git repository", exc_info=True) + slogger.task[tid].error("error has been occured during updating repository request", exc_info=True) + except: + pass + return HttpResponseBadRequest(str(e)) + return HttpResponse() + + +@login_required +@permission_required(perm=['engine.view_task'], raise_exception=True) +def push_repository(request, tid): + try: + slogger.task[tid].info("push repository request") + CVATGit.push(tid, request.user, request.scheme, request.get_host()) + except Exception as e: + try: + slogger.task[tid].error("error has been occured during pushing repository request", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) @@ -55,11 +70,11 @@ def update_repository(request): @permission_required(perm=['engine.view_task'], raise_exception=True) def get_repository(request, tid): try: - slogger.glob.info("get repository request for task #{}".format(tid)) + slogger.task[tid].info("get repository request") return JsonResponse(CVATGit.get(tid, request.user)) except Exception as e: try: - slogger.task[tid].error("error has been occured during getting repository info", exc_info=True) + slogger.task[tid].error("error has been occured during getting repository info request", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) @@ -69,11 +84,11 @@ def get_repository(request, tid): @permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) def delete_repository(request, tid): try: - slogger.glob.info("delete repository request for task #{}".format(tid)) + slogger.task[tid].info("delete repository request") CVATGit.delete(tid, request.user) except Exception as e: try: - slogger.task[tid].error("error has been occured during deleting a repository", exc_info=True) + slogger.task[tid].error("error has been occured during deleting repository request", exc_info=True) except: pass return HttpResponseBadRequest(str(e)) From 96542fb9b32310306c470c13c05470d5c93d5503 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 31 Oct 2018 18:10:53 +0300 Subject: [PATCH 15/84] Fixed heads --- cvat/apps/git/git.py | 3 ++- cvat/apps/git/urls.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index a0b60a0d6129..ad480e0a5369 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -207,10 +207,11 @@ def push(self, scheme, host, format): # Remove user branch from local repository if it exists if self.__user.username in list(map(lambda x: x.name, self.__rep.heads)): - git.Head.delete(self.__rep.heads[self.__user.username].repo, self.__rep.heads[self.__user.username]) + self.__rep.delete_head(self.__user.username, force=True) # Create new user branch from master self.__rep.create_head(self.__user.username) + self.__rep.head.reference = self.__rep.heads[self.__user.username] # Dump and zip dump(self.__tid, format, scheme, host) diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py index 24c3a08a8273..2fe3050e7614 100644 --- a/cvat/apps/git/urls.py +++ b/cvat/apps/git/urls.py @@ -13,4 +13,4 @@ path('get/', views.get_repository), path('delete/', views.delete_repository), path('push/', views.push_repository), -] \ No newline at end of file +] From 345afa762b2663806ae1f8f32f84d0a70ee8a547 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 1 Nov 2018 12:46:37 +0300 Subject: [PATCH 16/84] Push in client, diffs --- cvat/apps/engine/__init__.py | 2 - cvat/apps/engine/annotation.py | 2 + cvat/apps/engine/plugins.py | 52 +++++++++++++++ cvat/apps/git/git.py | 63 +++++++++++++------ .../apps/git/static/git/js/dashboardPlugin.js | 29 ++++++--- 5 files changed, 117 insertions(+), 31 deletions(-) create mode 100644 cvat/apps/engine/plugins.py diff --git a/cvat/apps/engine/__init__.py b/cvat/apps/engine/__init__.py index d8e62e54b356..b66dde17a5cf 100644 --- a/cvat/apps/engine/__init__.py +++ b/cvat/apps/engine/__init__.py @@ -1,5 +1,3 @@ - # Copyright (C) 2018 Intel Corporation # # SPDX-License-Identifier: MIT - diff --git a/cvat/apps/engine/annotation.py b/cvat/apps/engine/annotation.py index b5dc93a0567e..b0553687dda8 100644 --- a/cvat/apps/engine/annotation.py +++ b/cvat/apps/engine/annotation.py @@ -20,6 +20,7 @@ from django.db import transaction from cvat.apps.profiler import silk_profile +from cvat.apps.engine.plugins import plugin_decorator from . import models from .task import get_frame_path, get_image_meta_cache from .log import slogger @@ -72,6 +73,7 @@ def get(jid): return annotation.to_client() @silk_profile(name="Save job") +@plugin_decorator @transaction.atomic def save_job(jid, data): """ diff --git a/cvat/apps/engine/plugins.py b/cvat/apps/engine/plugins.py new file mode 100644 index 000000000000..da8f00697d13 --- /dev/null +++ b/cvat/apps/engine/plugins.py @@ -0,0 +1,52 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +__plugins = {} + +def add_plugin(name, function, order): + if order not in ["before", "after"]: + raise Exception("Order may be 'before' or 'after' only. Got {}.".format(order)) + + if not callable(function): + raise Exception("'function' argument should be a callable element") + + if not isinstance(name, str): + raise Exception("'name' argument should be a string. Got {}.".format(type(name))) + + if name not in __plugins: + __plugins[name] = { + "before": [], + "after": [] + } + + if function in __plugins[name][order]: + raise Exception("plugin already was attached") + + __plugins[name][order].append(function) + + +def remove_plugin(name, function): + if name in __plugins: + if function in __plugins[name]["before"]: + __plugins[name]["before"].remove(function) + if function in __plugins[name]["after"]: + __plugins[name]["after"].remove(function) + + +def plugin_decorator(function_to_decorate): + name = function_to_decorate.__name__ + def function_wrapper(*args, **kwargs): + if name in __plugins: + for wrapper in __plugins[name]["before"]: + wrapper(*args, **kwargs) + + result = function_to_decorate(*args, **kwargs) + + if name in __plugins: + for wrapper in __plugins[name]["after"]: + wrapper(*args, **kwargs) + + return result + + return function_wrapper \ No newline at end of file diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index ad480e0a5369..200b180ce390 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -5,8 +5,9 @@ from django.db import transaction from cvat.apps.engine.log import slogger -from cvat.apps.engine.models import Task -from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML +from cvat.apps.engine.models import Task, Job +from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML, save_job +from cvat.apps.engine.plugins import add_plugin from cvat.apps.git.models import GitData import subprocess @@ -172,22 +173,6 @@ def reclone(self): self.clone() - def onsave(self, changes_dict): - diff_dir = os.path.join(self.__cwd, DIFF_DIR) - if os.path.isdir(diff_dir): - diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - last_num = 0 - for f in diff_files: - number = f.split("_")[0] - number = int(number) if number.isdigit() else last_num - last_num = number - - with open("{}_{}.diff".format(last_num + 1, self.__user.username), 'w') as f: - f.write(json.dumps(changes_dict)) - else: - raise Exception("Local repository isn't found") - - def pull(self): self.__rep.head.reference = self.__rep.heads["master"] remote_branches = [] @@ -226,7 +211,7 @@ def push(self, scheme, host, format): new_dump_name = os.path.join(base_path, os.path.basename(dump_name)) os.rename(dump_name, new_dump_name) archive_name = os.path.join(base_path, "annotation.zip") - subprocess.call('zip -r "{}" "{}"'.format(archive_name, new_dump_name), shell=True) + subprocess.call('zip -j -r "{}" "{}"'.format(archive_name, new_dump_name), shell=True) os.remove(new_dump_name) # Commit and push @@ -234,6 +219,9 @@ def push(self, scheme, host, format): self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user.username, datetime.datetime.now())) self.__rep.git.push("origin", self.__user.username, '--force') + shutil.rmtree(os.path.join(self.__cwd, DIFF_DIR), True) + os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) + def delete(self): if os.path.isdir(self.__cwd): @@ -244,7 +232,7 @@ def remote_status(self): diff_dir = os.path.join(self.__cwd, DIFF_DIR) if os.path.isdir(diff_dir): diffs = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs)) + diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == ".diff", diffs)) return "actual" if not len(diffs) else "obsolete" else: raise Exception("Local repository isn't found") @@ -305,6 +293,38 @@ def get(tid, user): return response +@transaction.atomic +def onsave(jid, data): + db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task + try: + db_git = GitData.objects.select_for_update().get(pk = db_task.id) + diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repository", DIFF_DIR) + if os.path.isdir(diff_dir): + updated = sum([ len(data["update"][key]) for key in data["update"] ]) + deleted = sum([ len(data["delete"][key]) for key in data["delete"] ]) + created = sum([ len(data["create"][key]) for key in data["create"] ]) + + if updated or deleted or created: + diff = { + "time": str(datetime.datetime.now()), + "update": {key: len(data["update"][key]) for key in data["update"].keys()}, + "delete": {key: len(data["delete"][key]) for key in data["delete"].keys()}, + "create": {key: len(data["create"][key]) for key in data["create"].keys()} + } + + diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + last_num = 0 + for f in diff_files: + number = f.split("_")[0] + number = int(number) if number.isdigit() else last_num + last_num = number + + with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f: + f.write(json.dumps(diff)) + except GitData.ObjectDoesNotExist: + pass + + @transaction.atomic def delete(tid, user): db_task = Task.objects.get(pk = tid) @@ -313,3 +333,6 @@ def delete(tid, user): db_git = GitData.objects.select_for_update().get(pk = db_task) Git(db_git.url, tid, user).delete() db_git.delete() + + +add_plugin("save_job", onsave, "after") diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index fde06da62dbe..04d8d32887de 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -35,14 +35,6 @@ window.cvat.git = { createURLInputTextId: 'gitCreateURLInputText', updateState: () => { - /* Used unicode characters: - updating: ⏲ - pushed: ✓ - actual: ★ - obsolete: ☆ - error: ⚠ - */ - let gitWindow = $(`#${window.cvat.git.reposWindowId}`); let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`); let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`); @@ -295,6 +287,25 @@ document.addEventListener("DOMContentLoaded", () => { }); repositoryPushButton.on('click', () => { - // to do + gitLabelMessage.css('color', '#cccc00').text('Pushing..'); + gitLabelStatus.css('color', '#cccc00').text('\u25cc'); + repositoryUpdateButton.attr("disabled", true); + repositoryPushButton.attr("disabled", true); + + $.ajax({ + url: '/git/repository/push/' + gitWindow.attr('current_tid'), + success: window.cvat.git.updateState, + error: () => { + try { + let message = `Error was occured during pushing an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + showMessage(message); + throw Error(message); + } + finally { + window.cvat.git.updateState(); + } + } + }) }); }); From e2d2a5333ce2d78ed7f517cea875439759331671 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 1 Nov 2018 14:48:00 +0300 Subject: [PATCH 17/84] Merge diffs --- cvat/apps/git/git.py | 96 ++++++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 200b180ce390..72710e66f917 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -34,7 +34,7 @@ class Git: def __init__(self, url, tid, user): self.__url = url self.__tid = tid - self.__user = user + self.__user = '_{}'.format(user.username) self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repository") @@ -71,7 +71,7 @@ def _create_master(self): with open(readme_md_name, "w"): pass self.__rep.index.add([readme_md_name]) - self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user.username, datetime.datetime.now())) + self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) def init_repos(self): @@ -187,16 +187,37 @@ def pull(self): def push(self, scheme, host, format): + def _accumulate(source, target, target_key): + if isinstance(source, dict): + if target_key is not None and target_key not in target: + target[target_key] = {} + + for key in source: + if target_key is not None: + _accumulate(source[key], target[target_key], key) + else: + _accumulate(source[key], target, key) + elif isinstance(source, int): + if source: + if target_key is not None and target_key not in target: + target[target_key] = 0 + target[target_key] += source + else: + raise Exception("Unhandled accumulate type: {}".format(type(source))) + # Update local repository self.pull() # Remove user branch from local repository if it exists - if self.__user.username in list(map(lambda x: x.name, self.__rep.heads)): - self.__rep.delete_head(self.__user.username, force=True) + if self.__user in list(map(lambda x: x.name, self.__rep.heads)): + self.__rep.delete_head(self.__user, force=True) + self.__rep.head.reference = self.__rep.heads["master"] + self.__rep.head.reset('HEAD', index=True, working_tree=True) # Create new user branch from master - self.__rep.create_head(self.__user.username) - self.__rep.head.reference = self.__rep.heads[self.__user.username] + + self.__rep.create_head(self.__user) + self.__rep.head.reference = self.__rep.heads[self.__user] # Dump and zip dump(self.__tid, format, scheme, host) @@ -214,15 +235,28 @@ def push(self, scheme, host, format): subprocess.call('zip -j -r "{}" "{}"'.format(archive_name, new_dump_name), shell=True) os.remove(new_dump_name) + diff_path = os.path.join(self.__cwd, DIFF_DIR) + summary_diff = {} + for diff_name in list(map(lambda x: os.path.join(diff_path, x), os.listdir(diff_path))): + with open(diff_name, 'r') as f: + diff = json.loads(f.read()) + _accumulate(diff, summary_diff, None) + + diff_name = os.path.join(self.__cwd, "changelog.diff") + mode = 'a' if os.path.isfile(diff_name) else 'w' + with open(diff_name, mode) as f: + f.write('\n{}\n'.format(datetime.datetime.now())) + f.write(json.dumps(summary_diff)) + # Commit and push + self.__rep.index.add([diff_name]) self.__rep.index.add([archive_name]) - self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user.username, datetime.datetime.now())) - self.__rep.git.push("origin", self.__user.username, '--force') + self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user, datetime.datetime.now())) + self.__rep.git.push("origin", self.__user, '--force') shutil.rmtree(os.path.join(self.__cwd, DIFF_DIR), True) os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) - def delete(self): if os.path.isdir(self.__cwd): shutil.rmtree(self.__cwd) @@ -299,28 +333,28 @@ def onsave(jid, data): try: db_git = GitData.objects.select_for_update().get(pk = db_task.id) diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repository", DIFF_DIR) - if os.path.isdir(diff_dir): - updated = sum([ len(data["update"][key]) for key in data["update"] ]) - deleted = sum([ len(data["delete"][key]) for key in data["delete"] ]) - created = sum([ len(data["create"][key]) for key in data["create"] ]) - - if updated or deleted or created: - diff = { - "time": str(datetime.datetime.now()), - "update": {key: len(data["update"][key]) for key in data["update"].keys()}, - "delete": {key: len(data["delete"][key]) for key in data["delete"].keys()}, - "create": {key: len(data["create"][key]) for key in data["create"].keys()} - } - - diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - last_num = 0 - for f in diff_files: - number = f.split("_")[0] - number = int(number) if number.isdigit() else last_num - last_num = number - - with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f: - f.write(json.dumps(diff)) + os.makedirs(diff_dir, exist_ok = True) + + updated = sum([ len(data["update"][key]) for key in data["update"] ]) + deleted = sum([ len(data["delete"][key]) for key in data["delete"] ]) + created = sum([ len(data["create"][key]) for key in data["create"] ]) + + if updated or deleted or created: + diff = { + "update": {key: len(data["update"][key]) for key in data["update"].keys()}, + "delete": {key: len(data["delete"][key]) for key in data["delete"].keys()}, + "create": {key: len(data["create"][key]) for key in data["create"].keys()} + } + + diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + last_num = 0 + for f in diff_files: + number = os.path.splitext(os.path.basename(f))[0] + number = int(number) if number.isdigit() else last_num + last_num = max(last_num, number) + + with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f: + f.write(json.dumps(diff)) except GitData.ObjectDoesNotExist: pass From bf430bdc8456f73e4b17b3b89f2b0ad611f13dd0 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 1 Nov 2018 17:28:04 +0300 Subject: [PATCH 18/84] Async requests --- .../apps/git/static/git/js/dashboardPlugin.js | 188 ++++++++++++------ cvat/apps/git/urls.py | 1 + cvat/apps/git/views.py | 68 +++++-- 3 files changed, 173 insertions(+), 84 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 04d8d32887de..30a275f7fd22 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -92,75 +92,98 @@ window.cvat.git = { getGitURL: (success, error) => { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); - $.ajax({ - url: '/git/repository/get/' + gitWindow.attr('current_tid'), - type: 'GET', - success: success, - error: error - }); + $.get(`/git/repository/get/${gitWindow.attr('current_tid')}`).done( + success + ).fail(error); }, removeGitURL: () => { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); - $.ajax({ - url: '/git/repository/delete/' + gitWindow.attr('current_tid'), - type: 'DELETE', - success: window.cvat.git.updateState, - error: (data) => { - let message = `Error was occured during deleting an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } + $.get(`/git/repository/delete/${gitWindow.attr('current_tid')}`).done( + window.cvat.git.updateState + ).fail(() => { + let message = `Error was occured during deleting an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + window.cvat.git.badSituation(message); }); }, updateGitURL: (url) => { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); - $.ajax({ + $.post({ url: '/git/repository/update', - type: 'POST', data: JSON.stringify({ 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - success: window.cvat.git.updateState, - error: (data) => { - try { - let message = `Error was occured during updating an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - finally { - window.cvat.git.updateState(); - } - } + contentType: 'application/json;charset=utf-8', + }).done((data) => { + let checkInterval = setInterval(() => { + $.get(`/git/repository/check/${data.rq_id}`).done((data) => { + if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { + clearInterval(checkInterval); + if (data.status == "failed" || data.status == "unknown") { + let message = `Check request for git repostory returned "${data.status}" status`; + window.cvat.git.badSituation(message); + } + window.cvat.git.updateState(); + } + }).fail((data) => { + let message = `Check request for git repository failed. ` + + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`; + clearInterval(checkInterval); + window.cvat.git.badSituation(message); + }); + }, 1000); + }).fail((data) => { + let message = `Error was occured during updating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + window.cvat.git.badSituation(message); }); }, createGitURL: (url) => { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); - $.ajax({ + $.post({ url: '/git/repository/create', - type: 'POST', data: JSON.stringify({ 'tid': +gitWindow.attr('current_tid'), 'url': url, }), - success: window.cvat.git.updateState, - error: (data) => { - try { - let message = `Error was occured during creating an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - finally { - window.cvat.git.updateState(); - } - } + contentType: 'application/json;charset=utf-8', + }).done((data) => { + let checkInterval = setInterval(() => { + $.get(`/git/repository/check/${data.rq_id}`).done((data) => { + if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { + clearInterval(checkInterval); + if (data.status == "failed" || data.status == "unknown") { + let message = `Check request for git repostory returned "${data.status}" status`; + window.cvat.git.badSituation(message); + } + window.cvat.git.updateState(); + } + }).fail((data) => { + let message = `Check request for git repository failed. ` + + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`; + clearInterval(checkInterval); + window.cvat.git.badSituation(message); + }); + }, 1000); + }).fail((data) => { + let message = `Error was occured during creating an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + window.cvat.git.badSituation(message); }); + }, + + badSituation: (message) => { + try { + showMessage(message); + throw Error(message); + } + finally { + window.cvat.git.updateState(); + } } } @@ -182,21 +205,34 @@ document.addEventListener("DOMContentLoaded", () => { let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value').replace(/\s/g,''); if (gitURL.length) { - $.ajax({ - type: 'POST', + $.post({ url: '/git/repository/create', data: JSON.stringify({ 'tid': tid, 'url': gitURL, }), contentType: 'application/json;charset=utf-8', - error: (data) => { - console.log(`Warning: Can't create git record for task ${tid}. ` + - `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); - }, - complete: () => { - originalOnSuccessCreate(); - } + }).done((data) => { + let checkInterval = setInterval(() => { + $.get(`/git/repository/check/${data.rq_id}`).done((data) => { + if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { + clearInterval(checkInterval); + if (data.status == "failed" || data.status == "unknown") { + console.log(`Warning: Checking request for git repository returned status "${data.status}"`); + } + originalOnSuccessCreate(); + } + }).fail((data) => { + console.log(`Warning: Checking request for git repository failed. ` + + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); + clearInterval(checkInterval); + originalOnSuccessCreate(); + }); + }, 1000); + }).fail((data) => { + console.log(`Warning: Creation request for git repository failed. ` + + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); + originalOnSuccessCreate(); }); } else { @@ -292,20 +328,40 @@ document.addEventListener("DOMContentLoaded", () => { repositoryUpdateButton.attr("disabled", true); repositoryPushButton.attr("disabled", true); - $.ajax({ - url: '/git/repository/push/' + gitWindow.attr('current_tid'), - success: window.cvat.git.updateState, - error: () => { - try { + + $.get(`/git/repository/push/${gitWindow.attr('current_tid')}`) + .done((data) => { + let checkInterval = setInterval(() => { + $.get(`/git/repository/check/${data.rq_id}`).done((data) => { + if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { + clearInterval(checkInterval); + if (data.status == "failed" || data.status == "unknown") { + let message = `Pushing process returned "${data.status}" status`; + badSituation(message); + } + window.cvat.git.updateState(); + } + }).fail((data) => { + clearInterval(checkInterval); let message = `Error was occured during pushing an repos entry. ` + - `Code: ${data.status}, text: ${data.responseText || data.statusText}`; - showMessage(message); - throw Error(message); - } - finally { - window.cvat.git.updateState(); - } + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + badSituation(message); + }); + }, 1000); + }).fail((data) => { + let message = `Error was occured during pushing an repos entry. ` + + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; + badSituation(message); + }); + + function badSituation(message) { + try { + showMessage(message); + throw Error(message); + } + finally { + window.cvat.git.updateState(); } - }) + } }); }); diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py index 2fe3050e7614..fb44526c3ba5 100644 --- a/cvat/apps/git/urls.py +++ b/cvat/apps/git/urls.py @@ -13,4 +13,5 @@ path('get/', views.get_repository), path('delete/', views.delete_repository), path('push/', views.push_repository), + path('check/', views.check_process), ] diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 2bea3132bd6e..4a502dcc0159 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -10,10 +10,31 @@ from cvat.apps.engine.log import slogger import cvat.apps.git.git as CVATGit +import django_rq import json import git +@login_required +def check_process(request, rq_id): + try: + queue = django_rq.get_queue('default') + rq_job = queue.fetch_job(rq_id) + + if rq_job is not None: + if rq_job.is_queued or rq_job.is_started: + return JsonResponse({"status": "processing"}) + elif rq_job.is_finished: + return JsonResponse({"status": "finished"}) + else: + return JsonResponse({"status": "failed"}) + else: + return JsonResponse({"status": "unknown"}) + except Exception as ex: + slogger.glob.error("error has been occured during checking repository request with rq id {}".format(rq_id), exc_info=True) + return HttpResponseBadRequest(str(ex)) + + @login_required @permission_required('engine.add_task', raise_exception=True) def create_repository(request): @@ -24,11 +45,15 @@ def create_repository(request): url = data['url'] slogger.glob.info("create repository request for task #{}".format(tid)) - CVATGit.create(url, tid, request.user) - except Exception as e: + + rq_id = "git.create.{}".format(tid) + queue = django_rq.get_queue('default') + queue.enqueue_call(func = CVATGit.create, args = (url, tid, request.user), job_id= rq_id) + + return JsonResponse({ "rq_id": rq_id }) + except Exception as ex: slogger.glob.error("error has been occured during creating repository request for the task #{}".format(tid), exc_info=True) - return HttpResponseBadRequest(str(e)) - return HttpResponse() + return HttpResponseBadRequest(str(ex)) @login_required @@ -41,14 +66,18 @@ def update_repository(request): url = data['url'] slogger.task[tid].info("update repository request") - CVATGit.update(url, tid, request.user) - except Exception as e: + + rq_id = "git.update.{}".format(tid) + queue = django_rq.get_queue('default') + queue.enqueue_call(func = CVATGit.update, args = (url, tid, request.user), job_id = rq_id) + + return JsonResponse({ "rq_id": rq_id }) + except Exception as ex: try: slogger.task[tid].error("error has been occured during updating repository request", exc_info=True) except: pass - return HttpResponseBadRequest(str(e)) - return HttpResponse() + return HttpResponseBadRequest(str(ex)) @login_required @@ -56,14 +85,18 @@ def update_repository(request): def push_repository(request, tid): try: slogger.task[tid].info("push repository request") - CVATGit.push(tid, request.user, request.scheme, request.get_host()) - except Exception as e: + + rq_id = "git.push.{}".format(tid) + queue = django_rq.get_queue('default') + queue.enqueue_call(func = CVATGit.push, args = (tid, request.user, request.scheme, request.get_host()), job_id = rq_id) + + return JsonResponse({ "rq_id": rq_id }) + except Exception as ex: try: slogger.task[tid].error("error has been occured during pushing repository request", exc_info=True) except: pass - return HttpResponseBadRequest(str(e)) - return HttpResponse() + return HttpResponseBadRequest(str(ex)) @login_required @@ -72,12 +105,12 @@ def get_repository(request, tid): try: slogger.task[tid].info("get repository request") return JsonResponse(CVATGit.get(tid, request.user)) - except Exception as e: + except Exception as ex: try: slogger.task[tid].error("error has been occured during getting repository info request", exc_info=True) except: pass - return HttpResponseBadRequest(str(e)) + return HttpResponseBadRequest(str(ex)) @login_required @@ -86,11 +119,10 @@ def delete_repository(request, tid): try: slogger.task[tid].info("delete repository request") CVATGit.delete(tid, request.user) - except Exception as e: + return HttpResponse() + except Exception as ex: try: slogger.task[tid].error("error has been occured during deleting repository request", exc_info=True) except: pass - return HttpResponseBadRequest(str(e)) - - return HttpResponse() + return HttpResponseBadRequest(str(ex)) From f57f4ece0d3a14c4f797c35bde714aa7fd3ec1d4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 10:20:24 +0300 Subject: [PATCH 19/84] Updated plugin interface on the server, removed extra request to server during task creating --- cvat/apps/engine/plugins.py | 22 ++++++++-- cvat/apps/engine/task.py | 6 +-- cvat/apps/git/git.py | 12 +++++- .../apps/git/static/git/js/dashboardPlugin.js | 41 +------------------ 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/cvat/apps/engine/plugins.py b/cvat/apps/engine/plugins.py index da8f00697d13..0af2a517d0ce 100644 --- a/cvat/apps/engine/plugins.py +++ b/cvat/apps/engine/plugins.py @@ -2,9 +2,11 @@ # # SPDX-License-Identifier: MIT +from functools import update_wrapper + __plugins = {} -def add_plugin(name, function, order): +def add_plugin(name, function, order, exc_ok = False): if order not in ["before", "after"]: raise Exception("Order may be 'before' or 'after' only. Got {}.".format(order)) @@ -25,6 +27,8 @@ def add_plugin(name, function, order): __plugins[name][order].append(function) + function.exc_ok = exc_ok + def remove_plugin(name, function): if name in __plugins: @@ -36,17 +40,27 @@ def remove_plugin(name, function): def plugin_decorator(function_to_decorate): name = function_to_decorate.__name__ + def function_wrapper(*args, **kwargs): if name in __plugins: for wrapper in __plugins[name]["before"]: - wrapper(*args, **kwargs) + try: + wrapper(*args, **kwargs) + except Exception as ex: + if not wrapper.exc_ok: + raise ex result = function_to_decorate(*args, **kwargs) if name in __plugins: for wrapper in __plugins[name]["after"]: - wrapper(*args, **kwargs) + try: + wrapper(*args, **kwargs) + except Exception as ex: + if not wrapper.exc_ok: + raise ex return result - return function_wrapper \ No newline at end of file + update_wrapper(function_wrapper, function_to_decorate) + return function_wrapper diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 7a735ae901de..108a8ea68d5b 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -19,6 +19,7 @@ mimetypes.init(files=[_MEDIA_MIMETYPES_FILE]) from cvat.apps.engine.models import StatusChoice +from cvat.apps.engine.plugins import plugin_decorator import django_rq from django.conf import settings @@ -636,12 +637,9 @@ def _save_task_to_db(db_task, task_params): db_task.save() +@plugin_decorator @transaction.atomic def _create_thread(tid, params): - def raise_exception(images, dirs, videos, archives): - raise Exception('Only one archive, one video or many images can be dowloaded simultaneously. \ - {} image(s), {} dir(s), {} video(s), {} archive(s) found'.format(images, dirs, videos, archives)) - slogger.glob.info("create task #{}".format(tid)) job = rq.get_current_job() diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 72710e66f917..5c10fbd07956 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -331,7 +331,7 @@ def get(tid, user): def onsave(jid, data): db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task try: - db_git = GitData.objects.select_for_update().get(pk = db_task.id) + GitData.objects.select_for_update().get(pk = db_task.id) diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repository", DIFF_DIR) os.makedirs(diff_dir, exist_ok = True) @@ -369,4 +369,12 @@ def delete(tid, user): db_git.delete() -add_plugin("save_job", onsave, "after") +def _initial_create(tid, params): + url = params['git_url'] + user = params['owner'] + if len(url): + create(url, tid, user) + + +add_plugin("save_job", onsave, "after", exc_ok = False) +add_plugin("_create_thread", _initial_create, "after", exc_ok = True) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 30a275f7fd22..58dd5378c8da 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -200,45 +200,8 @@ document.addEventListener("DOMContentLoaded", () => { let originalCreateTaskRequest = window.createTaskRequest; window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) { try { - let originalOnSuccessCreate = onSuccessCreate; - onSuccessCreate = (tid) => { - let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value').replace(/\s/g,''); - - if (gitURL.length) { - $.post({ - url: '/git/repository/create', - data: JSON.stringify({ - 'tid': tid, - 'url': gitURL, - }), - contentType: 'application/json;charset=utf-8', - }).done((data) => { - let checkInterval = setInterval(() => { - $.get(`/git/repository/check/${data.rq_id}`).done((data) => { - if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { - clearInterval(checkInterval); - if (data.status == "failed" || data.status == "unknown") { - console.log(`Warning: Checking request for git repository returned status "${data.status}"`); - } - originalOnSuccessCreate(); - } - }).fail((data) => { - console.log(`Warning: Checking request for git repository failed. ` + - `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); - clearInterval(checkInterval); - originalOnSuccessCreate(); - }); - }, 1000); - }).fail((data) => { - console.log(`Warning: Creation request for git repository failed. ` + - `Status: ${data.status}. Message: ${data.responseText || data.statusText}`); - originalOnSuccessCreate(); - }); - } - else { - originalOnSuccessCreate(); - } - } + let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value').replace(/\s/g,''); + oData.append('git_url', gitURL); } finally { originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); From 807d9deb92371beee6e7340269261f28c64bbe32 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 11:52:52 +0300 Subject: [PATCH 20/84] Updated diffs dir, some refactoring --- cvat/apps/engine/plugins.py | 6 +- cvat/apps/git/git.py | 127 ++++++++++++++++++------------------ cvat/apps/git/settings.py | 9 +++ 3 files changed, 76 insertions(+), 66 deletions(-) create mode 100644 cvat/apps/git/settings.py diff --git a/cvat/apps/engine/plugins.py b/cvat/apps/engine/plugins.py index 0af2a517d0ce..6bf0143f39ce 100644 --- a/cvat/apps/engine/plugins.py +++ b/cvat/apps/engine/plugins.py @@ -6,6 +6,7 @@ __plugins = {} + def add_plugin(name, function, order, exc_ok = False): if order not in ["before", "after"]: raise Exception("Order may be 'before' or 'after' only. Got {}.".format(order)) @@ -23,7 +24,7 @@ def add_plugin(name, function, order, exc_ok = False): } if function in __plugins[name][order]: - raise Exception("plugin already was attached") + raise Exception("plugin has been attached already") __plugins[name][order].append(function) @@ -34,8 +35,10 @@ def remove_plugin(name, function): if name in __plugins: if function in __plugins[name]["before"]: __plugins[name]["before"].remove(function) + del function.exc_ok if function in __plugins[name]["after"]: __plugins[name]["after"].remove(function) + del function.exc_ok def plugin_decorator(function_to_decorate): @@ -62,5 +65,6 @@ def function_wrapper(*args, **kwargs): return result + # Copy meta info about wrapped function to wrapper function update_wrapper(function_wrapper, function_to_decorate) return function_wrapper diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 5c10fbd07956..d4e9d2d9d80a 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -8,7 +8,9 @@ from cvat.apps.engine.models import Task, Job from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML, save_job from cvat.apps.engine.plugins import add_plugin + from cvat.apps.git.models import GitData +from cvat.apps.git.settings import * import subprocess import datetime @@ -18,10 +20,6 @@ import re import os -DIFF_DIR = "cvat_diffs" -CVAT_USER = "cvat" -CVAT_EMAIL = "cvat@example.com" - class Git: __url = None @@ -29,15 +27,20 @@ class Git: __user = None __cwd = None __rep = None + __diffs_dir = None def __init__(self, url, tid, user): self.__url = url self.__tid = tid - self.__user = '_{}'.format(user.username) - self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repository") + self.__user = '{}_{}'.format(CVAT_BRANCH_PREFIX, user.username) + self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repos") + self.__diffs_dir = os.path.join(os.getcwd(), "data", str(tid), "repos_diffs") + # Method parses an got URL. + # SSH: git@github.com/proj/repos[.git] + # HTTP/HTTPS: [http://]github.com/proj/repos[.git] def _parse_url(self): http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)" ssh_pattern = "([a-zA-Z0-9._-]+)@([a-zA-Z0-9._-]+):([a-zA-Z.-]+)/([a-zA-Z.-]+)" @@ -62,9 +65,10 @@ def _parse_url(self): repos += ".git" return scheme, user, host, repos else: - raise Exception("Couldn't parse URL") + raise Exception("Got URL doesn't sutisfy for regular expression") + # Method creates the main branch if repostory don't have any branches def _create_master(self): assert not len(self.__rep.heads) readme_md_name = os.path.join(self.__cwd, "README.md") @@ -74,6 +78,8 @@ def _create_master(self): self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) + # Method connects local report if it exists + # Otherwise it clones it before def init_repos(self): try: # Try to use a local repos. It can throw GitError exception @@ -85,7 +91,7 @@ def init_repos(self): # We need reinitialize repository if it's false raise git.exc.GitError except git.exc.GitError: - slogger.task[self.__tid].info("Local repository reinitialization..") + slogger.task[self.__tid].info("Local repository initialization..") shutil.rmtree(self.__cwd, True) self.clone() @@ -104,6 +110,8 @@ def http_url(self): return "{}://{}/{}".format(scheme, host, repos) + # Method clones a remote repos to the local storage using SSH + # Method also initializes a repos after it has been cloned def clone(self): os.makedirs(self.__cwd) ssh_url = self.ssh_url() @@ -112,33 +120,31 @@ def clone(self): slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url)) self.__rep = git.Repo.clone_from(ssh_url, self.__cwd) - # Setup config file for CVAT user + # Setup config file for CVAT_HEADLESS user slogger.task[self.__tid].info("User config initialization..") with self.__rep.config_writer() as cw: if not cw.has_section("user"): cw.add_section("user") - cw.set("user", "name", CVAT_USER) - cw.set("user", "email", CVAT_EMAIL) + cw.set("user", "name", CVAT_HEADLESS_USERNAME) + cw.set("user", "email", CVAT_HEADLESS_EMAIL) cw.release() - # Create master branch if it doesn't exist and push it into server for remote repository initialization + # Create master branch if it doesn't exist + # And push it into server for a remote repos initialization if not len(self.__rep.heads): self._create_master() try: self.__rep.git.push("origin", "master") except git.exc.GitError: slogger.task[self.__tid].warning("Remote repository doesn't contain any " + - "heads but master push process is fault", exc_info = True) + "heads but 'master' branch push process has been failed", exc_info = True) - # Create dir for diffs and add it into .gitignore - os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) - gitignore = os.path.join(self.__cwd, ".gitignore") - file_mode = "a" if os.path.isfile(os.path.join(self.__cwd, ".gitignore")) else "w" - with open(gitignore, file_mode) as gitignore: - gitignore.writelines(["\n", "{}/".format(DIFF_DIR), "\n"]) + os.makedirs(self.__diffs_dir, exist_ok = True) - # Reclone method instead of clone try to save all accumulated diffs + # Method is some wrapper for clone + # It restores state if any errors have occured + # It useful if merge conflicts have occured during pull def reclone(self): if os.path.exists(self.__cwd): if not os.path.isdir(self.__cwd): @@ -147,32 +153,23 @@ def reclone(self): # Rename current repository dir tmp_repo = os.path.join(self.__cwd, "..", "tmp_repo") os.rename(self.__cwd, tmp_repo) - successful_cloning = False + # Try clone repository try: self.clone() - successful_cloning = True + shutil.rmtree(tmp_repo, True) except Exception as ex: - # Restore state if any error has been occured + # Restore state if any errors have occured if os.path.isdir(self.__cwd): shutil.rmtree(self.__cwd, True) os.rename(tmp_repo, self.__cwd) raise ex - - # If clone process is successful, push all diffs into new local repository - if successful_cloning: - if os.path.exists(os.path.join(tmp_repo, DIFF_DIR)): - diffs_to_move = list(map(lambda x: os.path.join(DIFF_DIR, x), os.listdir(os.path.join(tmp_repo, DIFF_DIR)))) - diffs_to_move = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == "diff", diffs_to_move)) - - for diff in diffs_to_move: - os.rename(os.path.join(tmp_repo, diff), os.path.join(self.__cwd, diff)) - shutil.rmtree(tmp_repo, True) return self.clone() + # Method checkout to master branch and pull it from remote repos def pull(self): self.__rep.head.reference = self.__rep.heads["master"] remote_branches = [] @@ -186,7 +183,10 @@ def pull(self): self.reclone() + # Method prepares an annotation, merges diffs and pushes it to remote repository to user branch def push(self, scheme, host, format): + + # Helpful function which merges diffs def _accumulate(source, target, target_key): if isinstance(source, dict): if target_key is not None and target_key not in target: @@ -214,39 +214,37 @@ def _accumulate(source, target, target_key): self.__rep.head.reference = self.__rep.heads["master"] self.__rep.head.reset('HEAD', index=True, working_tree=True) - # Create new user branch from master - + # Checkout new user branch from master self.__rep.create_head(self.__user) self.__rep.head.reference = self.__rep.heads[self.__user] - # Dump and zip + # Dump an annotation dump(self.__tid, format, scheme, host) - db_task = Task.objects.get(pk = self.__tid) - dump_name = db_task.get_dump_path() + dump_name = Task.objects.get(pk = self.__tid).get_dump_path() - if not os.path.isfile(dump_name): - raise Exception("Dump completed but file isn't found") + # Remove other annotation if it exists base_path = os.path.join(self.__cwd, "annotation") shutil.rmtree(base_path, True) - os.makedirs(base_path, exist_ok = True) - new_dump_name = os.path.join(base_path, os.path.basename(dump_name)) - os.rename(dump_name, new_dump_name) + os.makedirs(base_path) + + # Zip an annotation and remove it archive_name = os.path.join(base_path, "annotation.zip") - subprocess.call('zip -j -r "{}" "{}"'.format(archive_name, new_dump_name), shell=True) - os.remove(new_dump_name) + subprocess.call('zip -j -r "{}" "{}"'.format(archive_name, dump_name), shell=True) + os.remove(dump_name) - diff_path = os.path.join(self.__cwd, DIFF_DIR) + # Merge diffs summary_diff = {} - for diff_name in list(map(lambda x: os.path.join(diff_path, x), os.listdir(diff_path))): + for diff_name in list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))): with open(diff_name, 'r') as f: diff = json.loads(f.read()) _accumulate(diff, summary_diff, None) + # Save merged diffs file diff_name = os.path.join(self.__cwd, "changelog.diff") mode = 'a' if os.path.isfile(diff_name) else 'w' with open(diff_name, mode) as f: f.write('\n{}\n'.format(datetime.datetime.now())) - f.write(json.dumps(summary_diff)) + f.write(json.dumps(summary_diff, sort_keys = True, indent = 4)) # Commit and push self.__rep.index.add([diff_name]) @@ -254,22 +252,21 @@ def _accumulate(source, target, target_key): self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user, datetime.datetime.now())) self.__rep.git.push("origin", self.__user, '--force') - shutil.rmtree(os.path.join(self.__cwd, DIFF_DIR), True) - os.makedirs(os.path.join(self.__cwd, DIFF_DIR)) + shutil.rmtree(self.__diffs_dir, True) + + # Method deletes a local repos def delete(self): if os.path.isdir(self.__cwd): shutil.rmtree(self.__cwd) + # Method checks status of repository annotation def remote_status(self): - diff_dir = os.path.join(self.__cwd, DIFF_DIR) - if os.path.isdir(diff_dir): - diffs = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) - diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == ".diff", diffs)) - return "actual" if not len(diffs) else "obsolete" - else: - raise Exception("Local repository isn't found") + os.makedirs(self.__diffs_dir) + diffs = list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))) + diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == ".diff", diffs)) + return "actual" if not len(diffs) else "obsolete" @transaction.atomic @@ -292,6 +289,13 @@ def create(url, tid, user): raise ex +def _initial_create(tid, params): + url = params['git_url'] + user = params['owner'] + if len(url): + create(url, tid, user) + + @transaction.atomic def update(url, tid, user): db_task = Task.objects.get(pk = tid) @@ -332,7 +336,7 @@ def onsave(jid, data): db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task try: GitData.objects.select_for_update().get(pk = db_task.id) - diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repository", DIFF_DIR) + diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repos_diffs") os.makedirs(diff_dir, exist_ok = True) updated = sum([ len(data["update"][key]) for key in data["update"] ]) @@ -369,12 +373,5 @@ def delete(tid, user): db_git.delete() -def _initial_create(tid, params): - url = params['git_url'] - user = params['owner'] - if len(url): - create(url, tid, user) - - add_plugin("save_job", onsave, "after", exc_ok = False) add_plugin("_create_thread", _initial_create, "after", exc_ok = True) diff --git a/cvat/apps/git/settings.py b/cvat/apps/git/settings.py new file mode 100644 index 000000000000..3d650567a96b --- /dev/null +++ b/cvat/apps/git/settings.py @@ -0,0 +1,9 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os + +CVAT_BRANCH_PREFIX = "cvat" +CVAT_HEADLESS_USERNAME = os.environ['CVAT_HEADLESS_NAME'] +CVAT_HEADLESS_EMAIL = os.environ['CVAT_HEADLESS_EMAIL'] \ No newline at end of file From 23321098160b871503a77b007bb53e1b2fa2f24c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 12:20:32 +0300 Subject: [PATCH 21/84] Changed diff structure --- cvat/apps/engine/annotation.py | 8 +++++++- cvat/apps/git/git.py | 21 ++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cvat/apps/engine/annotation.py b/cvat/apps/engine/annotation.py index b0553687dda8..1bc95c2f68c6 100644 --- a/cvat/apps/engine/annotation.py +++ b/cvat/apps/engine/annotation.py @@ -91,7 +91,13 @@ def save_job(jid, data): annotation.save_to_db(data['create']) annotation.update_in_db(data['update']) - db_job.segment.task.updated_date = timezone.now() + updated = sum([ len(data["update"][key]) for key in data["update"] ]) + deleted = sum([ len(data["delete"][key]) for key in data["delete"] ]) + created = sum([ len(data["create"][key]) for key in data["create"] ]) + + if updated or deleted or created or delete_old_data: + db_job.segment.task.updated_date = timezone.now() + db_job.segment.task.save() db_job.max_shape_id = max(db_job.max_shape_id, max(client_ids['create']) if client_ids['create'] else -1) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index d4e9d2d9d80a..359232b34a65 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT from django.db import transaction +from django.utils import timezone from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Task, Job @@ -239,12 +240,22 @@ def _accumulate(source, target, target_key): diff = json.loads(f.read()) _accumulate(diff, summary_diff, None) + summary_diff["timestamp"] = timezone.now() + # Save merged diffs file diff_name = os.path.join(self.__cwd, "changelog.diff") - mode = 'a' if os.path.isfile(diff_name) else 'w' - with open(diff_name, mode) as f: - f.write('\n{}\n'.format(datetime.datetime.now())) - f.write(json.dumps(summary_diff, sort_keys = True, indent = 4)) + old_changes = [] + + if os.path.isfile(diff_name): + with open(diff_name, 'r') as f: + try: + old_changes = json.loads(f.read()) + except json.decoder.JSONDecodeError: + pass + + old_changes.append(summary_diff) + with open(diff_name, 'w') as f: + f.write(json.dumps(old_changes), sort_keys = True, indent = 4) # Commit and push self.__rep.index.add([diff_name]) @@ -263,7 +274,7 @@ def delete(self): # Method checks status of repository annotation def remote_status(self): - os.makedirs(self.__diffs_dir) + os.makedirs(self.__diffs_dir, exist_ok = True) diffs = list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))) diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == ".diff", diffs)) return "actual" if not len(diffs) else "obsolete" From 4bb94aa9ebc0a1f18b866daab1ae5320b685fb06 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 12:52:26 +0300 Subject: [PATCH 22/84] Check of empty repos --- cvat/apps/git/git.py | 29 ++++++++++++++++--- .../apps/git/static/git/js/dashboardPlugin.js | 5 ++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 359232b34a65..79ebe656e026 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -4,6 +4,7 @@ from django.db import transaction from django.utils import timezone +from django.utils.dateparse import parse_datetime from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Task, Job @@ -240,7 +241,7 @@ def _accumulate(source, target, target_key): diff = json.loads(f.read()) _accumulate(diff, summary_diff, None) - summary_diff["timestamp"] = timezone.now() + summary_diff["timestamp"] = str(timezone.now()) # Save merged diffs file diff_name = os.path.join(self.__cwd, "changelog.diff") @@ -255,7 +256,7 @@ def _accumulate(source, target, target_key): old_changes.append(summary_diff) with open(diff_name, 'w') as f: - f.write(json.dumps(old_changes), sort_keys = True, indent = 4) + f.write(json.dumps(old_changes, sort_keys = True, indent = 4)) # Commit and push self.__rep.index.add([diff_name]) @@ -273,7 +274,27 @@ def delete(self): # Method checks status of repository annotation - def remote_status(self): + def remote_status(self, last_save): + anno_archive_name = os.path.join(self.__cwd, "annotation", "annotation.zip") + changelog_name = os.path.join(self.__cwd, "changelog.diff") + + # Check repository exists and archive exists + if not os.path.isfile(anno_archive_name) or not os.path.isfile(changelog_name): + return "empty" + else: + with open(changelog_name, 'r') as f: + try: + data = json.loads(f.read()) + last_push = data[-1]["timestamp"] + last_push = parse_datetime(last_push) + if last_save > last_push: + return "obsolete" + else: + return "actual" + except json.decoder.JSONDecodeError: + raise Exception("Bad local repository.") + + # Check accumulated diffs os.makedirs(self.__diffs_dir, exist_ok = True) diffs = list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))) diffs = list(filter(lambda x: len(os.path.splitext(x)) > 1 and os.path.splitext(x)[1] == ".diff", diffs)) @@ -336,7 +357,7 @@ def get(tid, user): db_git = GitData.objects.select_for_update().get(pk = db_task) response['url']['value'] = db_git.url try: - response['status']['value'] = Git(db_git.url, tid, user).remote_status() + response['status']['value'] = Git(db_git.url, tid, user).remote_status(db_task.updated_date) except Exception as ex: response['status']['error'] = str(ex) return response diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 58dd5378c8da..20b255e0c73a 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -74,6 +74,11 @@ window.cvat.git = { gitLabelMessage.css('color', 'black').text('Repository contains obsolete data'); pushButton.attr("disabled", false); } + else if (data.status.value == "empty") { + gitLabelStatus.css('color', 'red').text('\u2606'); + gitLabelMessage.css('color', 'red').text('Empty local repository'); + pushButton.attr("disabled", false); + } else { let message = `Got unknown repository status: ${data.status.value}`; gitLabelStatus.css('color', 'red').text('\u26a0'); From 7f0f4669a9e96a622e91bcc8b600bfd22a0b3a88 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 13:36:29 +0300 Subject: [PATCH 23/84] Initialization of installation --- .gitignore | 1 + Dockerfile | 7 +++++++ components/README.md | 1 + components/git/README.md | 0 components/git/docker-compose.git.yml | 13 +++++++++++++ components/git/install.sh | 11 +++++++++++ components/ssh/README.md | 0 components/ssh/docker-compose.ssh.yml | 13 +++++++++++++ components/ssh/install.sh | 12 ++++++++++++ cvat/apps/git/settings.py | 2 +- cvat/apps/git/views.py | 1 + 11 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 components/git/README.md create mode 100644 components/git/docker-compose.git.yml create mode 100755 components/git/install.sh create mode 100644 components/ssh/README.md create mode 100644 components/ssh/docker-compose.ssh.yml create mode 100755 components/ssh/install.sh diff --git a/.gitignore b/.gitignore index cb48028971f0..f324eb840af9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ /logs /components/openvino/*.tgz /profiles +/components/ssh/keys/* # Ignore temporary files docker-compose.override.yml diff --git a/Dockerfile b/Dockerfile index 0f5ab6feda7b..da836c10966c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,6 +74,13 @@ RUN if [ "$TF_ANNOTATION" = "yes" ]; then \ bash -i /tmp/components/tf_annotation/install.sh; \ fi +# Custom SSH keys installation +ARG CUSTOM_SSH +ENV CUSTOM_SSH=${CUSTOM_SSH} +RUN if [ "$CUSTOM_SSH" = "yes" ]; then \ + bash -i /tmp/components/ssh/install.sh; \ + fi + ARG WITH_TESTS RUN if [ "$WITH_TESTS" = "yes" ]; then \ wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ diff --git a/components/README.md b/components/README.md index f02bee3b1c63..c207e206c798 100644 --- a/components/README.md +++ b/components/README.md @@ -4,3 +4,4 @@ * [OpenVINO](openvino/README.md) * [Tensorflow Object Detector](tf_annotation/README.md) * [Analytics](analytics/README.md) +* [SSH](analytics/README.md) diff --git a/components/git/README.md b/components/git/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/git/docker-compose.git.yml b/components/git/docker-compose.git.yml new file mode 100644 index 000000000000..af34067bebb8 --- /dev/null +++ b/components/git/docker-compose.git.yml @@ -0,0 +1,13 @@ +# +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +version: "2.3" + +services: + cvat: + build: + context: . + args: + GIT_SUPPORT: "yes" diff --git a/components/git/install.sh b/components/git/install.sh new file mode 100755 index 000000000000..55803fe22107 --- /dev/null +++ b/components/git/install.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +pip3 install GitPython +# Set env variables to bash: +# CVAT_HEADLESS_USERNAME +# CVAT_HEADLESS_EMAIL diff --git a/components/ssh/README.md b/components/ssh/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/ssh/docker-compose.ssh.yml b/components/ssh/docker-compose.ssh.yml new file mode 100644 index 000000000000..6dbb038084da --- /dev/null +++ b/components/ssh/docker-compose.ssh.yml @@ -0,0 +1,13 @@ +# +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +version: "2.3" + +services: + cvat: + build: + context: . + args: + CUSTOM_SSH: "yes" diff --git a/components/ssh/install.sh b/components/ssh/install.sh new file mode 100755 index 000000000000..afc57dad20a3 --- /dev/null +++ b/components/ssh/install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +set -e + +rm ${HOME}/.ssh -fr +mkdir ${HOME}/.ssh -p +mv /tmp/components/ssh/keys/* ${HOME}/.ssh/ diff --git a/cvat/apps/git/settings.py b/cvat/apps/git/settings.py index 3d650567a96b..339aa86d40d1 100644 --- a/cvat/apps/git/settings.py +++ b/cvat/apps/git/settings.py @@ -6,4 +6,4 @@ CVAT_BRANCH_PREFIX = "cvat" CVAT_HEADLESS_USERNAME = os.environ['CVAT_HEADLESS_NAME'] -CVAT_HEADLESS_EMAIL = os.environ['CVAT_HEADLESS_EMAIL'] \ No newline at end of file +CVAT_HEADLESS_EMAIL = os.environ['CVAT_HEADLESS_EMAIL'] diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 4a502dcc0159..0ecd53bbe3b9 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -10,6 +10,7 @@ from cvat.apps.engine.log import slogger import cvat.apps.git.git as CVATGit + import django_rq import json import git From 9758e9022074396190c41fea767cb42a50e12ac4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 15:58:16 +0300 Subject: [PATCH 24/84] Some bugs were fixed --- Dockerfile | 7 +++++++ components/git/docker-compose.git.yml | 3 +++ components/git/install.sh | 8 ++++---- cvat/apps/engine/views.py | 2 ++ cvat/apps/git/git.py | 15 ++++++++++++++- cvat/apps/git/settings.py | 2 +- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index da836c10966c..430fa9b83512 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,6 +81,13 @@ RUN if [ "$CUSTOM_SSH" = "yes" ]; then \ bash -i /tmp/components/ssh/install.sh; \ fi +# GIT integration support +ARG GIT_SUPPORT +ENV GIT_SUPPORT=${GIT_SUPPORT} +RUN if [ "$GIT_SUPPORT" = "yes" ]; then \ + bash -i /tmp/components/git/install.sh; \ + fi + ARG WITH_TESTS RUN if [ "$WITH_TESTS" = "yes" ]; then \ wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ diff --git a/components/git/docker-compose.git.yml b/components/git/docker-compose.git.yml index af34067bebb8..2bb857b325ed 100644 --- a/components/git/docker-compose.git.yml +++ b/components/git/docker-compose.git.yml @@ -11,3 +11,6 @@ services: context: . args: GIT_SUPPORT: "yes" + environment: + CVAT_HEADLESS_USERNAME: "cvat" + CVAT_HEADLESS_EMAIL: "cvat@example.com" diff --git a/components/git/install.sh b/components/git/install.sh index 55803fe22107..13eb955c54e9 100755 --- a/components/git/install.sh +++ b/components/git/install.sh @@ -5,7 +5,7 @@ # SPDX-License-Identifier: MIT # -pip3 install GitPython -# Set env variables to bash: -# CVAT_HEADLESS_USERNAME -# CVAT_HEADLESS_EMAIL +set -e + +pip3 install --no-cache-dir GitPython==2.1.11 +apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 4d4ac7b37f5f..a1c7fb1bc22b 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -22,6 +22,8 @@ from .log import slogger, clogger from cvat.apps.engine.models import StatusChoice + + ############################# High Level server API @login_required @permission_required(perm=['engine.job.access'], diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 79ebe656e026..4c3f05c9490e 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -80,6 +80,15 @@ def _create_master(self): self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user, datetime.datetime.now())) + def _init_host(self): + user, host = self._parse_url()[1:-1] + check_command = 'ssh-keygen -F {} | grep "Host {} found"'.format(host, host) + add_command = 'ssh -o StrictHostKeyChecking=no {}@{}'.format(user, host) + if not len(subprocess.run([check_command], shell = True, stdout=subprocess.PIPE).stdout): + subprocess.run([add_command], shell = True) + slogger.task[self.__tid].info('Host {} has been added to known_hosts.'.format(host)) + + # Method connects local report if it exists # Otherwise it clones it before def init_repos(self): @@ -120,6 +129,7 @@ def clone(self): # Clone repository slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url)) + self._init_host() self.__rep = git.Repo.clone_from(ssh_url, self.__cwd) # Setup config file for CVAT_HEADLESS user @@ -179,6 +189,7 @@ def pull(self): remote_branches.append(remote_branch.split("/")[-1]) if "master" in remote_branches: try: + self._init_host() self.__rep.git.pull("origin", "master") except git.exc.GitError: # Merge conflicts @@ -262,7 +273,9 @@ def _accumulate(source, target, target_key): self.__rep.index.add([diff_name]) self.__rep.index.add([archive_name]) self.__rep.index.commit("CVAT Annotation. Annotation updated by {} at {}".format(self.__user, datetime.datetime.now())) - self.__rep.git.push("origin", self.__user, '--force') + + self._init_host() + self.__rep.git.push("origin", self.__user, "--force") shutil.rmtree(self.__diffs_dir, True) diff --git a/cvat/apps/git/settings.py b/cvat/apps/git/settings.py index 339aa86d40d1..b169c88b5d64 100644 --- a/cvat/apps/git/settings.py +++ b/cvat/apps/git/settings.py @@ -5,5 +5,5 @@ import os CVAT_BRANCH_PREFIX = "cvat" -CVAT_HEADLESS_USERNAME = os.environ['CVAT_HEADLESS_NAME'] +CVAT_HEADLESS_USERNAME = os.environ['CVAT_HEADLESS_USERNAME'] CVAT_HEADLESS_EMAIL = os.environ['CVAT_HEADLESS_EMAIL'] From ad51be573878d2d2bef83a517277db839c72658f Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Nov 2018 16:02:09 +0300 Subject: [PATCH 25/84] eslint fixes --- cvat/apps/git/static/git/js/dashboardPlugin.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 20b255e0c73a..2045406ba886 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -106,7 +106,7 @@ window.cvat.git = { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); $.get(`/git/repository/delete/${gitWindow.attr('current_tid')}`).done( window.cvat.git.updateState - ).fail(() => { + ).fail((data) => { let message = `Error was occured during deleting an repos entry. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; window.cvat.git.badSituation(message); @@ -190,7 +190,7 @@ window.cvat.git = { window.cvat.git.updateState(); } } -} +}; document.addEventListener("DOMContentLoaded", () => { @@ -211,7 +211,7 @@ document.addEventListener("DOMContentLoaded", () => { finally { originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); } - } + }; /* GIT MODAL WINDOW PLUGIN PART */ $(` - + @@ -283,7 +204,7 @@ document.addEventListener("DOMContentLoaded", () => { let gitWindow = $(`#${window.cvat.git.reposWindowId}`); let closeRepositoryWindowButton = $(`#${window.cvat.git.closeReposWindowButtonId}`); - let repositoryPushButton = $(`#${window.cvat.git.reposPushButtonId}`); + let repositorySyncButton = $(`#${window.cvat.git.reposSyncButtonId}`); let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`); let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`); @@ -291,10 +212,10 @@ document.addEventListener("DOMContentLoaded", () => { gitWindow.addClass('hidden'); }); - repositoryPushButton.on('click', () => { + repositorySyncButton.on('click', () => { gitLabelMessage.css('color', '#cccc00').text('Pushing..'); gitLabelStatus.css('color', '#cccc00').text('\u25cc'); - repositoryPushButton.attr("disabled", true); + repositorySyncButton.attr("disabled", true); $.get(`/git/repository/push/${gitWindow.attr('current_tid')}`).done((data) => { let checkInterval = setInterval(() => { diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py index fb44526c3ba5..a04ac1537584 100644 --- a/cvat/apps/git/urls.py +++ b/cvat/apps/git/urls.py @@ -9,9 +9,7 @@ urlpatterns = [ path('create', views.create_repository), - path('update', views.update_repository), path('get/', views.get_repository), - path('delete/', views.delete_repository), path('push/', views.push_repository), path('check/', views.check_process), ] diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py index 56592d10f6af..b520babb17a3 100644 --- a/cvat/apps/git/views.py +++ b/cvat/apps/git/views.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: MIT from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse -from django.contrib.auth.decorators import permission_required from django.db import transaction from cvat.apps.authentication.decorators import login_required @@ -40,7 +39,6 @@ def check_process(request, rq_id): @login_required -@permission_required('engine.add_task', raise_exception=True) def create_repository(request): try: data = json.loads(request.body.decode('utf-8')) @@ -71,7 +69,6 @@ def create_repository(request): @login_required -@permission_required(perm=['engine.view_task'], raise_exception=True) def push_repository(request, tid): try: slogger.task[tid].info("push repository request") @@ -90,7 +87,6 @@ def push_repository(request, tid): @login_required -@permission_required(perm=['engine.view_task'], raise_exception=True) def get_repository(request, tid): try: slogger.task[tid].info("get repository request") @@ -101,42 +97,3 @@ def get_repository(request, tid): except: pass return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) -def update_repository(request): - try: - tid = None - data = json.loads(request.body.decode('utf-8')) - tid = data['tid'] - url = data['url'] - - slogger.task[tid].info("update repository request") - - rq_id = "git.update.{}".format(tid) - queue = django_rq.get_queue('default') - queue.enqueue_call(func = CVATGit.update, args = (url, tid, request.user), job_id = rq_id) - - return JsonResponse({ "rq_id": rq_id }) - except Exception as ex: - try: - slogger.task[tid].error("error has been occured during updating repository request", exc_info=True) - except: - pass - return HttpResponseBadRequest(str(ex)) - - -@login_required -@permission_required(perm=['engine.view_task', 'engine.change_task'], raise_exception=True) -def delete_repository(request, tid): - try: - slogger.task[tid].info("delete repository request") - CVATGit.delete(tid, request.user) - return HttpResponse() - except Exception as ex: - try: - slogger.task[tid].error("error has been occured during deleting repository request", exc_info=True) - except: - pass - return HttpResponseBadRequest(str(ex)) From 941a1a6bc165547b995963caa75e1a78f07c3920 Mon Sep 17 00:00:00 2001 From: bsekachev Date: Tue, 27 Nov 2018 09:08:16 +0300 Subject: [PATCH 43/84] Updated client part --- .../apps/git/static/git/js/dashboardPlugin.js | 124 ++++++++++-------- 1 file changed, 69 insertions(+), 55 deletions(-) diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js index 2f644e183598..2f727b9231b9 100644 --- a/cvat/apps/git/static/git/js/dashboardPlugin.js +++ b/cvat/apps/git/static/git/js/dashboardPlugin.js @@ -14,13 +14,12 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) { let elem = $(newElements[idx]); let tid = +elem.attr('id').split('_')[1]; - $('').addClass('regular dashboardButtonUI') - .on('click', () => { - let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); - gitDialogWindow.attr('current_tid', tid); - gitDialogWindow.removeClass('hidden'); - window.cvat.git.updateState(); - }).appendTo(elem.find('div.dashboardButtonsUI')[0]); + $('').addClass('regular dashboardButtonUI').on('click', () => { + let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`); + gitDialogWindow.attr('current_tid', tid); + gitDialogWindow.removeClass('hidden'); + window.cvat.git.updateState(); + }).appendTo(elem.find('div.dashboardButtonsUI')[0]); }); }); @@ -40,12 +39,14 @@ window.cvat.git = { let reposURLText = $(`#${window.cvat.git.reposURLTextId}`); let syncButton = $(`#${window.cvat.git.reposSyncButtonId}`); - reposURLText.prop('value', 'Getting an info..'); - gitLabelMessage.css('color', '#cccc00').text('Getting an info..'); + reposURLText.prop('value', 'Waiting for server response..'); + gitLabelMessage.css('color', '#cccc00').text('Waiting for server response..'); gitLabelStatus.css('color', '#cccc00').text('\u25cc'); syncButton.attr("disabled", true); - window.cvat.git.getGitURL((data) => { + let tid = gitWindow.attr('current_tid'); + + $.get(`/git/repository/get/${tid}`).done((data) => { if (!data.url.value) { gitLabelMessage.css('color', 'black').text('Repository is not attached'); reposURLText.attr('placeholder', 'Repository is not attached'); @@ -58,6 +59,7 @@ window.cvat.git = { if (!data.status.value) { gitLabelStatus.css('color', 'red').text('\u26a0'); gitLabelMessage.css('color', 'red').text(data.status.error); + syncButton.attr("disabled", false); return; } @@ -80,82 +82,77 @@ window.cvat.git = { gitLabelMessage.css('color', 'red').text(message); throw Error(message); } - }, (data) => { + }).fail(() => { gitWindow.addClass('hidden'); - let message = `Error was occured during get an repos URL. ` + + let message = `Error was occured during get an repos status. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; showMessage(message); throw Error(message); }); }, - - getGitURL: (success, error) => { - let gitWindow = $(`#${window.cvat.git.reposWindowId}`); - $.get(`/git/repository/get/${gitWindow.attr('current_tid')}`).done( - success - ).fail(error); - }, - - badSituation: (message) => { - try { - showMessage(message); - throw Error(message); - } - finally { - window.cvat.git.updateState(); - } - } }; document.addEventListener("DOMContentLoaded", () => { - /* CREATE TASK PLUGIN PART */ $(` - + `).insertAfter($("#dashboardBugTrackerInput").parent().parent()); + // Wrap create task request function let originalCreateTaskRequest = window.createTaskRequest; window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) { + // First we check if git URL specified let gitURL = $(`#${window.cvat.git.createURLInputTextId}`).prop('value').replace(/\s/g,''); if (gitURL.length) { + // If it specified, we try clone repository on a server side let taskMessage = $('#dashboardCreateTaskMessage'); taskMessage.css('color', 'green'); taskMessage.text('Cloning a repository..'); + $.post({ url: '/git/repository/create', data: JSON.stringify({ 'url': gitURL, }), contentType: 'application/json;charset=utf-8', - }).done((create_data) => { - let checkInterval = setInterval(() => { - $.get(`/git/repository/check/${create_data.rq_id}`).done((data) => { + }).done((createData) => { + setTimeout(timeoutCallback, 1000); + + function timeoutCallback() { + $.get(`/git/repository/check/${createData.rq_id}`).done((data) => { if (["finished", "failed", "unknown"].indexOf(data.status) != -1) { - clearInterval(checkInterval); if (data.status == "failed" || data.status == "unknown") { - let message = `Check request for git repostory returned "${data.status}" status`; + let message = `Request for verification of creation returned status "${data.status}"`; onError(`Git error. ${message}`); onComplete(); - return; } - oData.append('git_url', gitURL); - oData.append('repos_path', create_data['repos_path']); - originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); + else { + // Append some data for create task request if repository is successfully cloned + oData.append('git_url', gitURL); + oData.append('repos_path', createData['repos_path']); + + // And we perform original request function + originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus); + } + } + else { + setTimeout(timeoutCallback, 1000); } }).fail((data) => { - let message = `Check request for git repository failed. ` + + let message = `Request for verification of creation failed. ` + `Status: ${data.status}. Message: ${data.responseText || data.statusText}`; onError(`Git error. ${message}`); onComplete(); clearInterval(checkInterval); return; }); - }, 1000); + } }).fail((data) => { - let message = `Error was occured during updating an repos entry. ` + + let message = `Error occured during a request for verification of creation. ` + `Code: ${data.status}, text: ${data.responseText || data.statusText}`; onError(`Git error. ${message}`); onComplete(); @@ -169,7 +166,7 @@ document.addEventListener("DOMContentLoaded", () => { /* GIT MODAL WINDOW PLUGIN PART */ $(`