-
+
diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py
index 4d4ac7b37f5f..6c49e2842a54 100644
--- a/cvat/apps/engine/views.py
+++ b/cvat/apps/engine/views.py
@@ -104,8 +104,10 @@ def create_task(request):
return JsonResponse({'tid': db_task.id})
@login_required
-@permission_required(perm=['engine.task.access'],
- fn=objectgetter(models.Task, 'tid'), raise_exception=True)
+#@permission_required(perm=['engine.task.access'],
+# fn=objectgetter(models.Task, 'tid'), raise_exception=True)
+# We have commented lines above because the objectgetter() will raise 404 error in
+# cases when a task creating ends with an error. So an user don't get an actual reason of an error.
def check_task(request, tid):
"""Check the status of a task"""
try:
diff --git a/cvat/apps/git/README.md b/cvat/apps/git/README.md
new file mode 100644
index 000000000000..46a5629d8bfb
--- /dev/null
+++ b/cvat/apps/git/README.md
@@ -0,0 +1,17 @@
+## Git Integration For Annotation Storage
+
+### Description
+
+The application allows to integrate any git repository like an annotation storage for a CVAT task.
+It supports github or gitlab repositories.
+The SSH protocol is used for an authorization.
+
+### Using
+
+ * Put a private SSH key into the ```ssh``` directory. The public key corresponding to this private key should be attached to an github user.
+ * If you don't put any custom key, it will generated automatically.
+ * Setup a repository URL and a path (which is relative for a repository) in the create task dialog.
+ * Annotate a task.
+ * Press the button "Git Repository Sync" on the dashboard.
+ * In the dialog window press the button "Sync" and waiting for some time.
+ * An annotation will be dumped, archived and pushed to the attached remote repository. You can do a pull request manually.
diff --git a/cvat/apps/git/__init__.py b/cvat/apps/git/__init__.py
new file mode 100644
index 000000000000..a7027503b5db
--- /dev/null
+++ b/cvat/apps/git/__init__.py
@@ -0,0 +1,7 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from cvat.settings.base import JS_3RDPARTY
+
+JS_3RDPARTY['dashboard'] = JS_3RDPARTY.get('dashboard', []) + ['git/js/dashboardPlugin.js']
diff --git a/cvat/apps/git/admin.py b/cvat/apps/git/admin.py
new file mode 100644
index 000000000000..b66dde17a5cf
--- /dev/null
+++ b/cvat/apps/git/admin.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
diff --git a/cvat/apps/git/apps.py b/cvat/apps/git/apps.py
new file mode 100644
index 000000000000..9ef815879479
--- /dev/null
+++ b/cvat/apps/git/apps.py
@@ -0,0 +1,9 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.apps import AppConfig
+
+
+class GitConfig(AppConfig):
+ name = 'git'
diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py
new file mode 100644
index 000000000000..ab2f8d0a0410
--- /dev/null
+++ b/cvat/apps/git/git.py
@@ -0,0 +1,473 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.db import transaction
+from django.utils import timezone
+
+from cvat.apps.engine.log import slogger
+from cvat.apps.engine.models import Task, Job, User
+from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML
+from cvat.apps.engine.plugins import add_plugin
+from cvat.apps.git.models import GitStatusChoice
+
+from cvat.apps.git.models import GitData
+from collections import OrderedDict
+
+
+import subprocess
+import django_rq
+import shutil
+import json
+import git
+import os
+import re
+import rq
+
+def _have_no_access_exception(ex):
+ if 'Permission denied' in ex.stderr or 'Could not read from remote repository' in ex.stderr:
+ keys = subprocess.run(['ssh-add -L'], shell = True,
+ stdout = subprocess.PIPE).stdout.decode('utf-8').split('\n')
+ keys = list(filter(len, list(map(lambda x: x.strip(), keys))))
+ raise Exception(
+ 'Could not connect to the remote repository. ' +
+ 'Please make sure you have the correct access rights and the repository exists. ' +
+ 'Available public keys are: ' + str(keys)
+ )
+ else:
+ raise ex
+
+
+class Git:
+ __url = None
+ __path = None
+ __tid = None
+ __task_name = None
+ __branch_name = None
+ __user = None
+ __cwd = None
+ __rep = None
+ __diffs_dir = None
+ __annotation_file = None
+ __sync_date = None
+
+ def __init__(self, db_git, tid, user):
+ self.__db_git = db_git
+ self.__url = db_git.url
+ self.__path = db_git.path
+ self.__tid = tid
+ self.__user = {
+ "name": user.username,
+ "email": user.email or "dummy@cvat.com"
+ }
+ self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repos")
+ self.__diffs_dir = os.path.join(os.getcwd(), "data", str(tid), "repos_diffs")
+ self.__task_name = re.sub(r'[\\/*?:"<>|\s]', '_', Task.objects.get(pk = tid).name)[:100]
+ self.__branch_name = 'cvat_{}_{}'.format(tid, self.__task_name)
+ self.__annotation_file = os.path.join(self.__cwd, self.__path)
+ self.__sync_date = db_git.sync_date
+
+
+ # Method parses an got URL.
+ # SSH: git@github.com/proj/repos[.git]
+ # HTTP/HTTPS: [http://]github.com/proj/repos[.git]
+ def _parse_url(self):
+ try:
+ http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)"
+ ssh_pattern = "([a-zA-Z0-9._-]+)@([a-zA-Z0-9._-]+):([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)"
+
+ http_match = re.match(http_pattern, self.__url)
+ ssh_match = re.match(ssh_pattern, self.__url)
+
+ user = "git"
+ host = None
+ repos = None
+
+ if http_match:
+ host = http_match.group(2)
+ repos = "{}/{}".format(http_match.group(3), http_match.group(4))
+ elif ssh_match:
+ user = ssh_match.group(1)
+ host = ssh_match.group(2)
+ repos = "{}/{}".format(ssh_match.group(3), ssh_match.group(4))
+ else:
+ raise Exception("Got URL doesn't sutisfy for regular expression")
+
+ if not repos.endswith(".git"):
+ repos += ".git"
+
+ return user, host, repos
+ except Exception as ex:
+ slogger.glob.exception('URL parsing errors occured', exc_info = True)
+ raise ex
+
+
+ # Method creates the main branch if repostory doesn't have any branches
+ def _create_master_branch(self):
+ if len(self.__rep.heads):
+ raise Exception("Some heads already exists")
+ readme_md_name = os.path.join(self.__cwd, "README.md")
+ with open(readme_md_name, "w"):
+ pass
+ self.__rep.index.add([readme_md_name])
+ self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user["name"], timezone.now()))
+
+ self.__rep.git.push("origin", "master")
+
+
+ # Method creates task branch for repository from current master
+ def _to_task_branch(self):
+ # Remove user branch from local repository if it exists
+ if self.__branch_name not in list(map(lambda x: x.name, self.__rep.heads)):
+ self.__rep.create_head(self.__branch_name)
+
+ self.__rep.head.reference = self.__rep.heads[self.__branch_name]
+
+
+ # Method setups a config file for current user
+ def _update_config(self):
+ slogger.task[self.__tid].info("User config initialization..")
+ with self.__rep.config_writer() as cw:
+ if not cw.has_section("user"):
+ cw.add_section("user")
+ cw.set("user", "name", self.__user["name"])
+ cw.set("user", "email", self.__user["email"])
+ cw.release()
+
+ # Method initializes repos. It setup configuration, creates master branch if need and checkouts to task branch
+ def _configurate(self):
+ self._update_config()
+ if not len(self.__rep.heads):
+ self._create_master_branch()
+ self._to_task_branch()
+ os.makedirs(self.__diffs_dir, exist_ok = True)
+
+
+
+ def _ssh_url(self):
+ user, host, repos = self._parse_url()
+ return "{}@{}:{}".format(user, host, repos)
+
+
+ # Method clones a remote repos to the local storage using SSH and initializes it
+ def _clone(self):
+ os.makedirs(self.__cwd)
+ ssh_url = self._ssh_url()
+
+ # Cloning
+ slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url))
+ self.__rep = git.Repo.clone_from(ssh_url, self.__cwd)
+
+ # Intitialization
+ self._configurate()
+
+
+ # Method is some wrapper for clone
+ # It restores state if any errors have occured
+ # It useful if merge conflicts have occured during pull
+ def _reclone(self):
+ if os.path.exists(self.__cwd):
+ if not os.path.isdir(self.__cwd):
+ os.remove(self.__cwd)
+ else:
+ # Rename current repository dir
+ tmp_repo = os.path.abspath(os.path.join(self.__cwd, "..", "tmp_repo"))
+ os.rename(self.__cwd, tmp_repo)
+
+ # Try clone repository
+ try:
+ self._clone()
+ shutil.rmtree(tmp_repo, True)
+ except Exception as ex:
+ # Restore state if any errors have occured
+ if os.path.isdir(self.__cwd):
+ shutil.rmtree(self.__cwd, True)
+ os.rename(tmp_repo, self.__cwd)
+ raise ex
+ else:
+ self._clone()
+
+
+ # Method checkouts to master branch and pulls it from remote repos
+ def _pull(self):
+ self.__rep.head.reference = self.__rep.heads["master"]
+ try:
+ self.__rep.git.pull("origin", "master")
+
+ if self.__branch_name in list(map(lambda x: x.name, self.__rep.heads)):
+ self.__rep.head.reference = self.__rep.heads["master"]
+ self.__rep.delete_head(self.__branch_name, force=True)
+ self.__rep.head.reset("HEAD", index=True, working_tree=True)
+
+ self._to_task_branch()
+ except git.exc.GitError:
+ # Merge conflicts
+ self._reclone()
+
+
+ # Method connects a local repository if it exists
+ # Otherwise it clones it before
+ def init_repos(self, wo_remote = False):
+ try:
+ # Try to use a local repos. It can throw GitError exception
+ self.__rep = git.Repo(self.__cwd)
+ self._configurate()
+
+ # Check if remote URL is actual
+ if self._ssh_url() != self.__rep.git.remote('get-url', '--all', 'origin'):
+ slogger.task[self.__tid].info("Local repository URL is obsolete.")
+ # We need reinitialize repository if it's false
+ raise git.exc.GitError("Actual and saved repository URLs aren't match")
+ except git.exc.GitError:
+ if wo_remote:
+ raise Exception('Local repository is failed')
+ slogger.task[self.__tid].info("Local repository initialization..")
+ shutil.rmtree(self.__cwd, True)
+ self._clone()
+
+
+ # Method prepares an annotation, merges diffs and pushes it to remote repository to user branch
+ def push(self, scheme, host, format, last_save):
+
+ # Helpful function which merges diffs
+ def _accumulate(source, target, target_key):
+ if isinstance(source, dict):
+ if target_key is not None and target_key not in target:
+ target[target_key] = {}
+
+ for key in source:
+ if target_key is not None:
+ _accumulate(source[key], target[target_key], key)
+ else:
+ _accumulate(source[key], target, key)
+ elif isinstance(source, int):
+ if source:
+ if target_key is not None and target_key not in target:
+ target[target_key] = 0
+ target[target_key] += source
+ else:
+ raise Exception("Unhandled accumulate type: {}".format(type(source)))
+
+ # Update local repository
+ self._pull()
+
+ os.makedirs(os.path.join(self.__cwd, os.path.dirname(self.__annotation_file)), exist_ok = True)
+ # Remove old annotation file if it exists
+ if os.path.exists(self.__annotation_file):
+ os.remove(self.__annotation_file)
+
+ # Dump an annotation
+ dump(self.__tid, format, scheme, host, OrderedDict())
+ dump_name = Task.objects.get(pk = self.__tid).get_dump_path()
+
+ ext = os.path.splitext(self.__path)[1]
+ if ext == '.zip':
+ subprocess.call('zip -j -r "{}" "{}"'.format(self.__annotation_file, dump_name), shell=True)
+ elif ext == '.xml':
+ shutil.copyfile(dump_name, self.__annotation_file)
+ else:
+ raise Exception("Got unknown annotation file type")
+
+ # Setup LFS for *.zip files
+ self.__rep.git.lfs("track", self.__path)
+ self.__rep.git.add(self.__annotation_file)
+
+ # Merge diffs
+ summary_diff = {}
+ for diff_name in list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))):
+ with open(diff_name, 'r') as f:
+ diff = json.loads(f.read())
+ _accumulate(diff, summary_diff, None)
+
+ # Commit and push
+ self.__rep.index.add([
+ '.gitattributes',
+ ])
+ self.__rep.index.commit("CVAT Annotation updated by {}. Summary: {}".format(self.__user["name"], str(summary_diff)))
+
+ self.__rep.git.push("origin", self.__branch_name, "--force")
+
+ shutil.rmtree(self.__diffs_dir, True)
+
+
+ # Method checks status of repository annotation
+ def remote_status(self, last_save):
+ # Check repository exists and archive exists
+ if not os.path.isfile(self.__annotation_file) or last_save != self.__sync_date:
+ return GitStatusChoice.NON_SYNCED
+ else:
+ self.__rep.git.update_ref('-d', 'refs/remotes/origin/{}'.format(self.__branch_name))
+ self.__rep.git.remote('-v', 'update')
+
+ last_hash = self.__rep.git.show_ref('refs/heads/{}'.format(self.__branch_name), '--hash')
+ merge_base_hash = self.__rep.merge_base('refs/remotes/origin/master', self.__branch_name)[0].hexsha
+ if last_hash == merge_base_hash:
+ return GitStatusChoice.MERGED
+ else:
+ try:
+ self.__rep.git.show_ref('refs/remotes/origin/{}'.format(self.__branch_name), '--hash')
+ return GitStatusChoice.SYNCED
+ except git.exc.GitCommandError:
+ # Remote branch has been deleted w/o merge
+ return GitStatusChoice.NON_SYNCED
+
+
+def _initial_create(tid, params):
+ if 'git_path' in params:
+ try:
+ job = rq.get_current_job()
+ job.meta['status'] = 'Cloning a repository..'
+ job.save_meta()
+
+ user = params['owner']
+ git_path = params['git_path']
+
+ db_task = Task.objects.get(pk = tid)
+ path_pattern = r"\[(.+)\]"
+ path_search = re.search(path_pattern, git_path)
+ path = None
+
+ if path_search is not None:
+ path = path_search.group(1)
+ git_path = git_path[0:git_path.find(path) - 1].strip()
+ path = os.path.join('/', path.strip())
+ else:
+ anno_file = re.sub(r'[\\/*?:"<>|\s]', '_', db_task.name)[:100]
+ path = '/annotation/{}.zip'.format(anno_file)
+
+ path = path[1:]
+ _split = os.path.splitext(path)
+ if len(_split) < 2 or _split[1] not in [".xml", ".zip"]:
+ raise Exception("Only .xml and .zip formats are supported")
+
+ db_git = GitData()
+ db_git.url = git_path
+ db_git.path = path
+ db_git.task = db_task
+
+ try:
+ _git = Git(db_git, tid, user)
+ _git.init_repos()
+ db_git.save()
+ except git.exc.GitCommandError as ex:
+ _have_no_access_exception(ex)
+ except Exception as ex:
+ slogger.task[tid].exception('exception occured during git _initial_create', exc_info = True)
+ raise ex
+
+
+@transaction.atomic
+def push(tid, user, scheme, host):
+ try:
+ db_task = Task.objects.get(pk = tid)
+ db_git = GitData.objects.select_for_update().get(pk = db_task)
+ try:
+ _git = Git(db_git, tid, user)
+ _git.init_repos()
+ _git.push(scheme, host, FORMAT_XML, db_task.updated_date)
+
+ # Update timestamp
+ db_git.sync_date = db_task.updated_date
+ db_git.status = GitStatusChoice.SYNCED
+ db_git.save()
+ except git.exc.GitCommandError as ex:
+ _have_no_access_exception(ex)
+ except Exception as ex:
+ slogger.task[tid].exception('push to remote repository errors occured', exc_info = True)
+ raise ex
+
+
+@transaction.atomic
+def get(tid, user):
+ response = {}
+ response["url"] = {"value": None}
+ response["status"] = {"value": None, "error": None}
+
+ db_task = Task.objects.get(pk = tid)
+ if GitData.objects.filter(pk = db_task).exists():
+ db_git = GitData.objects.select_for_update().get(pk = db_task)
+ response['url']['value'] = '{} [{}]'.format(db_git.url, db_git.path)
+ try:
+ rq_id = "git.push.{}".format(tid)
+ queue = django_rq.get_queue('default')
+ rq_job = queue.fetch_job(rq_id)
+ if rq_job is not None and (rq_job.is_queued or rq_job.is_started):
+ db_git.status = GitStatusChoice.SYNCING
+ response['status']['value'] = str(db_git.status)
+ else:
+ try:
+ _git = Git(db_git, tid, user)
+ _git.init_repos(True)
+ db_git.status = _git.remote_status(db_task.updated_date)
+ response['status']['value'] = str(db_git.status)
+ except git.exc.GitCommandError as ex:
+ _have_no_access_exception(ex)
+ except Exception as ex:
+ db_git.status = GitStatusChoice.NON_SYNCED
+ response['status']['error'] = str(ex)
+
+ db_git.save()
+ return response
+
+def update_states():
+ db_git_records = GitData.objects.all()
+ db_user = User.objects.first()
+ if db_user is None:
+ # User hasn't been created yet
+ return
+
+ for db_git in db_git_records:
+ try:
+ get(db_git.task_id, db_user)
+ except Exception:
+ slogger.glob("Exception occured during a status updating for db_git with tid: {}".format(db_git.task_id))
+
+@transaction.atomic
+def _onsave(jid, data):
+ db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task
+ try:
+ db_git = GitData.objects.select_for_update().get(pk = db_task.id)
+ diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repos_diffs")
+ os.makedirs(diff_dir, exist_ok = True)
+
+ updated = sum([ len(data["update"][key]) for key in data["update"] ])
+ deleted = sum([ len(data["delete"][key]) for key in data["delete"] ])
+ created = sum([ len(data["create"][key]) for key in data["create"] ])
+
+ if updated or deleted or created:
+ diff = {
+ "update": {key: len(data["update"][key]) for key in data["update"].keys()},
+ "delete": {key: len(data["delete"][key]) for key in data["delete"].keys()},
+ "create": {key: len(data["create"][key]) for key in data["create"].keys()}
+ }
+
+ diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir)))
+ last_num = 0
+ for f in diff_files:
+ number = os.path.splitext(os.path.basename(f))[0]
+ number = int(number) if number.isdigit() else last_num
+ last_num = max(last_num, number)
+
+ with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f:
+ f.write(json.dumps(diff))
+
+ db_git.status = GitStatusChoice.NON_SYNCED
+ db_git.save()
+
+ except GitData.DoesNotExist:
+ pass
+
+def _ondump(tid, data_format, scheme, host, plugin_meta_data):
+ db_task = Task.objects.get(pk = tid)
+ try:
+ db_git = GitData.objects.get(pk = db_task)
+ plugin_meta_data['git'] = OrderedDict({
+ "url": db_git.url,
+ "path": db_git.path,
+ })
+ except GitData.DoesNotExist:
+ pass
+
+add_plugin("save_job", _onsave, "after", exc_ok = False)
+add_plugin("_create_thread", _initial_create, "before", exc_ok = False)
+add_plugin("_dump", _ondump, "before", exc_ok = False)
diff --git a/cvat/apps/git/management/__init__.py b/cvat/apps/git/management/__init__.py
new file mode 100644
index 000000000000..d89b5880c713
--- /dev/null
+++ b/cvat/apps/git/management/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
\ No newline at end of file
diff --git a/cvat/apps/git/management/commands/__init__.py b/cvat/apps/git/management/commands/__init__.py
new file mode 100644
index 000000000000..d89b5880c713
--- /dev/null
+++ b/cvat/apps/git/management/commands/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
\ No newline at end of file
diff --git a/cvat/apps/git/management/commands/update_git_states.py b/cvat/apps/git/management/commands/update_git_states.py
new file mode 100644
index 000000000000..68263eef7197
--- /dev/null
+++ b/cvat/apps/git/management/commands/update_git_states.py
@@ -0,0 +1,21 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.core.management.base import BaseCommand
+from cvat.apps.git.git import update_states
+import time
+
+INTERVAL_SEC = 600
+
+class Command(BaseCommand):
+ help = 'Run a regular updating for git status'
+
+ def handle(self, *args, **options):
+ while True:
+ try:
+ update_states()
+ except Exception as ex:
+ print("An error occured during update task statuses: {}".format(str(ex)))
+ time.sleep(INTERVAL_SEC)
+
diff --git a/cvat/apps/git/migrations/0001_initial.py b/cvat/apps/git/migrations/0001_initial.py
new file mode 100644
index 000000000000..10daa9c61e0b
--- /dev/null
+++ b/cvat/apps/git/migrations/0001_initial.py
@@ -0,0 +1,26 @@
+# Generated by Django 2.1.3 on 2018-12-05 13:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('engine', '0014_job_max_shape_id'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='GitData',
+ fields=[
+ ('task', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='engine.Task')),
+ ('url', models.URLField(max_length=2000)),
+ ('path', models.CharField(max_length=256)),
+ ('sync_date', models.DateTimeField(auto_now_add=True)),
+ ('status', models.CharField(default='!sync', max_length=20)),
+ ],
+ ),
+ ]
diff --git a/cvat/apps/git/migrations/__init__.py b/cvat/apps/git/migrations/__init__.py
new file mode 100644
index 000000000000..b66dde17a5cf
--- /dev/null
+++ b/cvat/apps/git/migrations/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
diff --git a/cvat/apps/git/models.py b/cvat/apps/git/models.py
new file mode 100644
index 000000000000..d72c921c057e
--- /dev/null
+++ b/cvat/apps/git/models.py
@@ -0,0 +1,24 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.db import models
+from cvat.apps.engine.models import Task
+from enum import Enum
+
+class GitStatusChoice(Enum):
+ NON_SYNCED = '!sync'
+ SYNCING = 'syncing'
+ SYNCED = 'sync'
+ MERGED = 'merged'
+
+ def __str__(self):
+ return self.value
+
+
+class GitData(models.Model):
+ task = models.OneToOneField(Task, on_delete = models.CASCADE, primary_key = True)
+ url = models.URLField(max_length = 2000)
+ path = models.CharField(max_length=256)
+ sync_date = models.DateTimeField(auto_now_add=True)
+ status = models.CharField(max_length=20, default=GitStatusChoice.NON_SYNCED)
diff --git a/cvat/apps/git/static/git/js/dashboardPlugin.js b/cvat/apps/git/static/git/js/dashboardPlugin.js
new file mode 100644
index 000000000000..31580bc2e046
--- /dev/null
+++ b/cvat/apps/git/static/git/js/dashboardPlugin.js
@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2018 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+"use strict";
+
+window.cvat = window.cvat || {};
+window.cvat.dashboard = window.cvat.dashboard || {};
+window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || [];
+window.cvat.dashboard.uiCallbacks.push(function(newElements) {
+ $.ajax({
+ type: "GET",
+ url: "/git/repository/meta/get",
+ success: (data) => {
+ newElements.each(function(idx) {
+ let elem = $(newElements[idx]);
+ let tid = +elem.attr("id").split("_")[1];
+
+ if (tid in data) {
+ if (["sync", "syncing"].includes(data[tid])) {
+ elem.css("background", "floralwhite");
+ }
+ else if (data[tid] === "merged") {
+ elem.css("background", "azure");
+ }
+ else {
+ elem.css("background", "mistyrose");
+ }
+
+ $("
").addClass("regular dashboardButtonUI").on("click", () => {
+ let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`);
+ gitDialogWindow.attr("current_tid", tid);
+ gitDialogWindow.removeClass("hidden");
+ window.cvat.git.updateState();
+ }).appendTo(elem.find("div.dashboardButtonsUI")[0]);
+ }
+ });
+ },
+ error: (data) => {
+ let message = `Can not get git repositories meta info. Code: ${data.status}. Message: ${data.responseText || data.statusText}`;
+ showMessage(message);
+ throw Error(message);
+ }
+ });
+
+
+});
+
+window.cvat.git = {
+ reposWindowId: "gitReposWindow",
+ closeReposWindowButtonId: "closeGitReposButton",
+ reposURLTextId: "gitReposURLText",
+ reposSyncButtonId: "gitReposSyncButton",
+ labelStatusId: "gitReposLabelStatus",
+ labelMessageId: "gitReposLabelMessage",
+ createURLInputTextId: "gitCreateURLInputText",
+
+ updateState: () => {
+ let gitWindow = $(`#${window.cvat.git.reposWindowId}`);
+ let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`);
+ let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`);
+ let reposURLText = $(`#${window.cvat.git.reposURLTextId}`);
+ let syncButton = $(`#${window.cvat.git.reposSyncButtonId}`);
+
+ reposURLText.attr("placeholder", "Waiting for server response..");
+ reposURLText.prop("value", "");
+ gitLabelMessage.css("color", "#cccc00").text("Waiting for server response..");
+ gitLabelStatus.css("color", "#cccc00").text("\u25cc");
+ syncButton.attr("disabled", true);
+
+ let tid = gitWindow.attr("current_tid");
+
+ $.get(`/git/repository/get/${tid}`).done(function(data) {
+ if (!data.url.value) {
+ gitLabelMessage.css("color", "black").text("Repository is not attached");
+ reposURLText.prop("value", "");
+ reposURLText.attr("placeholder", "Repository is not attached");
+ return;
+ }
+
+ reposURLText.attr("placeholder", "");
+ reposURLText.prop("value", data.url.value);
+
+ if (!data.status.value) {
+ gitLabelStatus.css("color", "red").text("\u26a0");
+ gitLabelMessage.css("color", "red").text(data.status.error);
+ syncButton.attr("disabled", false);
+ return;
+ }
+
+ if (data.status.value === "!sync") {
+ gitLabelStatus.css("color", "red").text("\u2606");
+ gitLabelMessage.css("color", "red").text("Repository is not synchronized");
+ syncButton.attr("disabled", false);
+ }
+ else if (data.status.value === "sync") {
+ gitLabelStatus.css("color", "#cccc00").text("\u2605");
+ gitLabelMessage.css("color", "black").text("Synchronized (merge required)");
+ }
+ else if (data.status.value === "merged") {
+ gitLabelStatus.css("color", "darkgreen").text("\u2605");
+ gitLabelMessage.css("color", "darkgreen").text("Synchronized");
+ }
+ else if (data.status.value === "syncing") {
+ gitLabelMessage.css("color", "#cccc00").text("Synchronization..");
+ gitLabelStatus.css("color", "#cccc00").text("\u25cc");
+ }
+ else {
+ let message = `Got unknown repository status: ${data.status.value}`;
+ gitLabelStatus.css("color", "red").text("\u26a0");
+ gitLabelMessage.css("color", "red").text(message);
+ throw Error(message);
+ }
+ }).fail(function(data) {
+ gitWindow.addClass("hidden");
+ let message = `Error occured during get an repos status. ` +
+ `Code: ${data.status}, text: ${data.responseText || data.statusText}`;
+ showMessage(message);
+ throw Error(message);
+ });
+ },
+};
+
+
+document.addEventListener("DOMContentLoaded", () => {
+ $(`
+
+ | |
+ ` +
+ ` |
+
`
+ ).insertAfter($("#dashboardBugTrackerInput").parent().parent());
+
+ // Wrap create task request function
+ let originalCreateTaskRequest = window.createTaskRequest;
+ window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) {
+ let gitPath = $(`#${window.cvat.git.createURLInputTextId}`).prop("value").replace(/\s/g, "");
+ if (gitPath.length) {
+ oData.append("git_path", gitPath);
+ }
+ originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus);
+ };
+
+ /* GIT MODAL WINDOW PLUGIN PART */
+ $(`
`).appendTo("body");
+
+ let gitWindow = $(`#${window.cvat.git.reposWindowId}`);
+ let closeRepositoryWindowButton = $(`#${window.cvat.git.closeReposWindowButtonId}`);
+ let repositorySyncButton = $(`#${window.cvat.git.reposSyncButtonId}`);
+ let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`);
+ let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`);
+
+ closeRepositoryWindowButton.on("click", () => {
+ gitWindow.addClass("hidden");
+ });
+
+ repositorySyncButton.on("click", () => {
+ function badResponse(message) {
+ try {
+ showMessage(message);
+ throw Error(message);
+ }
+ finally {
+ window.cvat.git.updateState();
+ }
+ }
+
+ gitLabelMessage.css("color", "#cccc00").text("Synchronization..");
+ gitLabelStatus.css("color", "#cccc00").text("\u25cc");
+ repositorySyncButton.attr("disabled", true);
+
+ let tid = gitWindow.attr("current_tid");
+ $.get(`/git/repository/push/${tid}`).done((data) => {
+ setTimeout(timeoutCallback, 1000);
+
+ function timeoutCallback() {
+ $.get(`/git/repository/check/${data.rq_id}`).done((data) => {
+ if (["finished", "failed", "unknown"].indexOf(data.status) != -1) {
+ if (data.status === "failed") {
+ let message = data.error;
+ badResponse(message);
+ }
+ else if (data.status === "unknown") {
+ let message = `Request for pushing returned status "${data.status}".`;
+ badResponse(message);
+ }
+ else {
+ window.cvat.git.updateState();
+ }
+ }
+ else {
+ setTimeout(timeoutCallback, 1000);
+ }
+ }).fail((data) => {
+ let message = `Error was occured during pushing an repos entry. ` +
+ `Code: ${data.status}, text: ${data.responseText || data.statusText}`;
+ badResponse(message);
+ });
+ }
+ }).fail((data) => {
+ let message = `Error was occured during pushing an repos entry. ` +
+ `Code: ${data.status}, text: ${data.responseText || data.statusText}`;
+ badResponse(message);
+ });
+ });
+});
diff --git a/cvat/apps/git/tests.py b/cvat/apps/git/tests.py
new file mode 100644
index 000000000000..3517aa28fb20
--- /dev/null
+++ b/cvat/apps/git/tests.py
@@ -0,0 +1,7 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/cvat/apps/git/urls.py b/cvat/apps/git/urls.py
new file mode 100644
index 000000000000..7a168290083b
--- /dev/null
+++ b/cvat/apps/git/urls.py
@@ -0,0 +1,15 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+
+from django.urls import path
+from . import views
+
+
+urlpatterns = [
+ path('get/
', views.get_repository),
+ path('push/', views.push_repository),
+ path('check/', views.check_process),
+ path('meta/get', views.get_meta_info),
+]
diff --git a/cvat/apps/git/views.py b/cvat/apps/git/views.py
new file mode 100644
index 000000000000..0365243460a3
--- /dev/null
+++ b/cvat/apps/git/views.py
@@ -0,0 +1,82 @@
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.http import HttpResponseBadRequest, JsonResponse
+from rules.contrib.views import permission_required, objectgetter
+
+from cvat.apps.authentication.decorators import login_required
+from cvat.apps.engine.log import slogger
+from cvat.apps.engine import models
+from cvat.apps.git.models import GitData
+
+import cvat.apps.git.git as CVATGit
+import django_rq
+
+@login_required
+def check_process(request, rq_id):
+ try:
+ queue = django_rq.get_queue('default')
+ rq_job = queue.fetch_job(rq_id)
+
+ if rq_job is not None:
+ if rq_job.is_queued or rq_job.is_started:
+ return JsonResponse({"status": "processing"})
+ elif rq_job.is_finished:
+ return JsonResponse({"status": "finished"})
+ else:
+ return JsonResponse({"status": "failed", "error": rq_job.exc_info})
+ else:
+ return JsonResponse({"status": "unknown"})
+ except Exception as ex:
+ slogger.glob.error("error occured during checking repository request with rq id {}".format(rq_id), exc_info=True)
+ return HttpResponseBadRequest(str(ex))
+
+
+@login_required
+@permission_required(perm=['engine.task.delete'],
+ fn=objectgetter(models.Task, 'tid'), raise_exception=True)
+def push_repository(request, tid):
+ try:
+ slogger.task[tid].info("push repository request")
+
+ rq_id = "git.push.{}".format(tid)
+ queue = django_rq.get_queue('default')
+ queue.enqueue_call(func = CVATGit.push, args = (tid, request.user, request.scheme, request.get_host()), job_id = rq_id)
+
+ return JsonResponse({ "rq_id": rq_id })
+ except Exception as ex:
+ try:
+ slogger.task[tid].error("error occured during pushing repository request", exc_info=True)
+ except Exception:
+ pass
+ return HttpResponseBadRequest(str(ex))
+
+
+@login_required
+@permission_required(perm=['engine.task.access'],
+ fn=objectgetter(models.Task, 'tid'), raise_exception=True)
+def get_repository(request, tid):
+ try:
+ slogger.task[tid].info("get repository request")
+ return JsonResponse(CVATGit.get(tid, request.user))
+ except Exception as ex:
+ try:
+ slogger.task[tid].error("error occured during getting repository info request", exc_info=True)
+ except Exception:
+ pass
+ return HttpResponseBadRequest(str(ex))
+
+
+@login_required
+def get_meta_info(request):
+ try:
+ db_git_records = GitData.objects.all()
+ response = {}
+ for db_git in db_git_records:
+ response[db_git.task_id] = db_git.status
+
+ return JsonResponse(response, safe = False)
+ except Exception as ex:
+ slogger.glob.exception("error occured during get meta request", exc_info = True)
+ return HttpResponseBadRequest(str(ex))
diff --git a/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js b/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js
index f7e0f3f0a03c..4deed70a88d7 100644
--- a/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js
+++ b/cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js
@@ -110,7 +110,7 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) {
let tfAnnotationButton = $('');
tfAnnotationButton.on('click', onTFAnnotationClick.bind(tfAnnotationButton));
- tfAnnotationButton.addClass('dashboardTFAnnotationButton semiBold dashboardButtonUI');
+ tfAnnotationButton.addClass('dashboardTFAnnotationButton regular dashboardButtonUI');
tfAnnotationButton.appendTo(buttonsUI);
if ((tid in data) && (data[tid].active)) {
diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt
index bf4e6c1e4a22..764503dcb3c0 100644
--- a/cvat/requirements/base.txt
+++ b/cvat/requirements/base.txt
@@ -25,3 +25,4 @@ dj-pagination==2.4.0
python-logstash==0.4.6
django-revproxy==0.9.15
rules==2.0
+GitPython==2.1.11
diff --git a/cvat/settings/base.py b/cvat/settings/base.py
index 246dd22ba81d..db89c9878a7d 100644
--- a/cvat/settings/base.py
+++ b/cvat/settings/base.py
@@ -16,6 +16,10 @@
import os
import sys
+import fcntl
+import shutil
+import subprocess
+
from pathlib import Path
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -34,6 +38,43 @@
f.write("SECRET_KEY = '{}'\n".format(get_random_string(50, chars)))
from keys.secret_key import SECRET_KEY
+
+def generate_ssh_keys():
+ keys_dir = '{}/keys'.format(os.getcwd())
+ ssh_dir = '{}/.ssh'.format(os.getenv('HOME'))
+ pidfile = os.path.join(ssh_dir, 'ssh.pid')
+
+ with open(pidfile, "w") as pid:
+ fcntl.flock(pid, fcntl.LOCK_EX)
+ try:
+ subprocess.run(['ssh-add {}/*'.format(ssh_dir)], shell = True, stderr = subprocess.PIPE)
+ keys = subprocess.run(['ssh-add -l'], shell = True,
+ stdout = subprocess.PIPE).stdout.decode('utf-8').split('\n')
+ if 'has no identities' in keys[0]:
+ print('SSH keys were not found')
+ volume_keys = os.listdir(keys_dir)
+ if not ('id_rsa' in volume_keys and 'id_rsa.pub' in volume_keys):
+ print('New pair of keys are being generated')
+ subprocess.run(['ssh-keygen -b 4096 -t rsa -f {}/id_rsa -q -N ""'.format(ssh_dir)], shell = True)
+ shutil.copyfile('{}/id_rsa'.format(ssh_dir), '{}/id_rsa'.format(keys_dir))
+ shutil.copymode('{}/id_rsa'.format(ssh_dir), '{}/id_rsa'.format(keys_dir))
+ shutil.copyfile('{}/id_rsa.pub'.format(ssh_dir), '{}/id_rsa.pub'.format(keys_dir))
+ shutil.copymode('{}/id_rsa.pub'.format(ssh_dir), '{}/id_rsa.pub'.format(keys_dir))
+ else:
+ print('Copying them from keys volume')
+ shutil.copyfile('{}/id_rsa'.format(keys_dir), '{}/id_rsa'.format(ssh_dir))
+ shutil.copymode('{}/id_rsa'.format(keys_dir), '{}/id_rsa'.format(ssh_dir))
+ shutil.copyfile('{}/id_rsa.pub'.format(keys_dir), '{}/id_rsa.pub'.format(ssh_dir))
+ shutil.copymode('{}/id_rsa.pub'.format(keys_dir), '{}/id_rsa.pub'.format(ssh_dir))
+ subprocess.run(['ssh-add', '{}/id_rsa'.format(ssh_dir)], shell = True)
+ finally:
+ fcntl.flock(pid, fcntl.LOCK_UN)
+
+try:
+ generate_ssh_keys()
+except Exception:
+ pass
+
# Application definition
JS_3RDPARTY = {}
@@ -48,6 +89,7 @@
'cvat.apps.dashboard',
'cvat.apps.authentication',
'cvat.apps.documentation',
+ 'cvat.apps.git',
'django_rq',
'compressor',
'cacheops',
diff --git a/cvat/urls.py b/cvat/urls.py
index 18744e6f9c60..3680a012ff8c 100644
--- a/cvat/urls.py
+++ b/cvat/urls.py
@@ -37,6 +37,9 @@
if apps.is_installed('cvat.apps.tf_annotation'):
urlpatterns.append(path('tensorflow/annotation/', include('cvat.apps.tf_annotation.urls')))
+if apps.is_installed('cvat.apps.git'):
+ urlpatterns.append(path('git/repository/', include('cvat.apps.git.urls')))
+
if apps.is_installed('cvat.apps.auto_annotation'):
urlpatterns.append(path('auto_annotation/', include('cvat.apps.auto_annotation.urls')))
diff --git a/docker-compose.yml b/docker-compose.yml
index 0f8bda786bf1..7e55bd3f1ed7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,6 +44,7 @@ services:
http_proxy:
https_proxy:
no_proxy:
+ socks_proxy:
TF_ANNOTATION: "no"
USER: "django"
DJANGO_CONFIGURATION: "production"
diff --git a/ssh/README.md b/ssh/README.md
new file mode 100644
index 000000000000..b5dcd609638e
--- /dev/null
+++ b/ssh/README.md
@@ -0,0 +1,22 @@
+Put your SSH keys and SSH config here and they will be installed to the CVAT container.
+
+If you have any problems with a git repository cloning inside the CVAT:
+ * Make sure that SSH keys have been added to the CVAT container:
+
+```bash
+docker exec -it cvat bash -ic 'ls .ssh'
+```
+
+ * If you need a proxy for connecting to the Internet, specify the socks_proxy variable before build the container. For example:
+
+```bash
+socks_proxy=proxy-example.com:1080 docker-compose build
+```
+
+ * Try to clone a repository via SSH directly in the container by the command:
+
+```bash
+docker exec -it cvat bash -ic 'cd /tmp -r && git clone '
+```
+
+ * Finally try to clone it on your local machine and if it's successful, contact with us via [Gitter chat](https://gitter.im/opencv-cvat) or [Github issues](https://github.com/opencv/cvat/issues).
diff --git a/supervisord.conf b/supervisord.conf
index 2aa7f92fd203..8ed246da09b4 100644
--- a/supervisord.conf
+++ b/supervisord.conf
@@ -17,15 +17,28 @@ loglevel=debug ; info, debug, warn, trace
pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live
+[program:ssh-agent]
+command=bash -c "rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-agent.sock"
+priority=1
+autorestart=true
+
[program:rqworker_default]
command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic \
"exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
+environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
numprocs=2
process_name=rqworker_default_%(process_num)s
[program:rqworker_low]
command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
"exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
+environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
+numprocs=1
+
+[program:git_status_updater]
+command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
+ "/usr/bin/python3 ~/manage.py update_git_states"
+environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
numprocs=1
[program:runserver]
@@ -40,3 +53,4 @@ command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \
exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
--limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \
%(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s"
+environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"