From 7ce6eb65d746d0f26befb2d930428c41b8b67a73 Mon Sep 17 00:00:00 2001 From: Boris Sekachev <40690378+bsekachev@users.noreply.github.com> Date: Thu, 4 Apr 2019 22:31:29 +0300 Subject: [PATCH] Integration of REST/API in client (Part 9) (#377) * Fixed: version of undefined * Fixed overlap * Fixed parser --- .../static/dashboard/js/dashboard.js | 6 +- cvat/apps/engine/annotation.py | 2 - cvat/apps/engine/annotation_v2.py | 4 +- .../migrations/0019_auto_20190404_1906.py | 18 ++ cvat/apps/engine/models.py | 2 +- .../static/engine/js/annotationParser.js | 17 +- cvat/apps/engine/task.py | 7 +- cvat/apps/git/git.py | 253 +++++++++--------- 8 files changed, 166 insertions(+), 143 deletions(-) create mode 100644 cvat/apps/engine/migrations/0019_auto_20190404_1906.py diff --git a/cvat/apps/dashboard/static/dashboard/js/dashboard.js b/cvat/apps/dashboard/static/dashboard/js/dashboard.js index abae5eefbd58..2b2451faad5e 100644 --- a/cvat/apps/dashboard/static/dashboard/js/dashboard.js +++ b/cvat/apps/dashboard/static/dashboard/js/dashboard.js @@ -103,7 +103,7 @@ class TaskView { if (next) { const response = await $.ajax({ - url: `/api/v1/tasks/${this._id}/annotations`, + url: `/api/v1/tasks/${this._id}/annotations?action=create`, type: 'PATCH', data: JSON.stringify(chunk), contentType: 'application/json', @@ -114,12 +114,12 @@ class TaskView { } async function save(parsed) {3 - const response = await $.ajax({ + await $.ajax({ url: `/api/v1/tasks/${this._id}/annotations`, type: 'DELETE', }); - await saveChunk.call(this, parsed, 0, response.version); + await saveChunk.call(this, parsed, 0, 0); } async function onload(overlay, text) { diff --git a/cvat/apps/engine/annotation.py b/cvat/apps/engine/annotation.py index cb132b68c321..7c7cad6063c0 100644 --- a/cvat/apps/engine/annotation.py +++ b/cvat/apps/engine/annotation.py @@ -73,7 +73,6 @@ def get(jid): return annotation.to_client() @silk_profile(name="Save job") -@plugin_decorator @transaction.atomic def save_job(jid, data): """ @@ -1504,7 +1503,6 @@ def init_from_db(self): self.points = annotation.points self.points_paths = annotation.points_paths -@plugin_decorator def _dump(tid, data_format, scheme, host, plugin_meta_data): # For big tasks dump function may run for a long time and # we dont need to acquire lock after _AnnotationForTask instance diff --git a/cvat/apps/engine/annotation_v2.py b/cvat/apps/engine/annotation_v2.py index 395c61aa482f..fea83a02e29b 100644 --- a/cvat/apps/engine/annotation_v2.py +++ b/cvat/apps/engine/annotation_v2.py @@ -56,6 +56,7 @@ def put_job_data(pk, data): return annotation.data @silk_profile(name="UPDATE job data") +@plugin_decorator @transaction.atomic def patch_job_data(pk, data, action): annotation = JobAnnotation(pk) @@ -110,6 +111,7 @@ def delete_task_data(pk): annotation = TaskAnnotation(pk) annotation.delete() + def dump_task_data(pk, file_path, scheme, host, query_params): # For big tasks dump function may run for a long time and # we dont need to acquire lock after _AnnotationForTask instance @@ -608,7 +610,7 @@ def init_from_db(self): self.db_task.overlap) self._merge_tracks(annotation.data["tracks"], db_segment.start_frame, self.db_task.overlap) - + def _merge_tags(self, tags, start_frame, overlap): # FIXME: implement merge algorithm here self.data["tags"].extend(tags) diff --git a/cvat/apps/engine/migrations/0019_auto_20190404_1906.py b/cvat/apps/engine/migrations/0019_auto_20190404_1906.py new file mode 100644 index 000000000000..5643f2327969 --- /dev/null +++ b/cvat/apps/engine/migrations/0019_auto_20190404_1906.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.7 on 2019-04-04 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0018_remove_job_max_shape_id'), + ] + + operations = [ + migrations.AlterField( + model_name='task', + name='overlap', + field=models.PositiveIntegerField(null=True), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 0e82f9799101..17f7b0e50e20 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -46,7 +46,7 @@ class Task(models.Model): bug_tracker = models.CharField(max_length=2000, blank=True, default="") created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now_add=True) - overlap = models.PositiveIntegerField(default=0) + overlap = models.PositiveIntegerField(null=True) # Zero means that there are no limits (default) segment_size = models.PositiveIntegerField(default=0) z_order = models.BooleanField(default=False) diff --git a/cvat/apps/engine/static/engine/js/annotationParser.js b/cvat/apps/engine/static/engine/js/annotationParser.js index 02ef224c9635..46d89366eb84 100644 --- a/cvat/apps/engine/static/engine/js/annotationParser.js +++ b/cvat/apps/engine/static/engine/js/annotationParser.js @@ -117,7 +117,7 @@ class AnnotationParser { _getAttributeList(shape, labelId) { let attributeDict = {}; let attributes = shape.getElementsByTagName('attribute'); - for (let attribute of attributes ) { + for (let attribute of attributes) { let [id, value] = this._getAttribute(labelId, attribute); attributeDict[id] = value; } @@ -314,6 +314,10 @@ class AnnotationParser { shapes: [], }; + if (path.frame < this._startFrame || path.frame > this._stopFrame) { + continue; + } + for (let shape of parsed[type]) { const keyFrame = +shape.getAttribute('keyframe'); const outside = +shape.getAttribute('outside'); @@ -325,7 +329,8 @@ class AnnotationParser { Ignore all frames less then start. Ignore all frames more then stop. */ - const significant = keyFrame || frame === this._startFrame; + const significant = (keyFrame || frame === this._startFrame) + && frame >= this._startFrame && frame <= this._stopFrame; if (significant) { const attributeList = this._getAttributeList(shape, labelId); @@ -333,23 +338,23 @@ class AnnotationParser { const pathAttributes = []; for (let attr of attributeList) { - const attrInfo = this._labelsInfo.attrInfo(attr.id); + const attrInfo = this._labelsInfo.attrInfo(attr.spec_id); if (attrInfo.mutable) { shapeAttributes.push({ - id: attr.id, + spec_id: attr.spec_id, value: attr.value, }); } else { pathAttributes.push({ - id: attr.id, + spec_id: attr.spec_id, value: attr.value, }); } } path.attributes = pathAttributes; - if (type === 'boxes') { + if (type === 'box') { let [points, occluded, z_order] = this._getBoxPosition(shape, Math.clamp(frame, this._startFrame, this._stopFrame)); path.shapes.push({ attributes: shapeAttributes, diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index d6a673d41003..8f7823698e33 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -239,11 +239,16 @@ def _save_task_to_db(db_task): job.meta['status'] = 'Task is being saved in database' job.save_meta() + default_overlap = 5 if db_task.mode == 'interpolation' else 0 + if db_task.overlap is None: + db_task.overlap = default_overlap + db_task.overlap = min(db_task.overlap, db_task.size - 1) + segment_size = db_task.segment_size if segment_size == 0: segment_size = db_task.size - for x in range(0, db_task.size, segment_size): + for x in range(0, db_task.size, segment_size - db_task.overlap): start_frame = x stop_frame = min(x + segment_size - 1, db_task.size - 1) slogger.glob.info("New segment for task #{}: start_frame = {}, \ diff --git a/cvat/apps/git/git.py b/cvat/apps/git/git.py index 96137c55937d..6a6eca156973 100644 --- a/cvat/apps/git/git.py +++ b/cvat/apps/git/git.py @@ -7,18 +7,19 @@ from cvat.apps.engine.log import slogger from cvat.apps.engine.models import Task, Job, User -from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML +from cvat.apps.engine.annotation_v2 import dump_task_data from cvat.apps.engine.plugins import add_plugin from cvat.apps.git.models import GitStatusChoice from cvat.apps.git.models import GitData from collections import OrderedDict - import subprocess import django_rq +import datetime import shutil import json +import math import git import os import re @@ -39,35 +40,22 @@ def _have_no_access_exception(ex): class Git: - __url = None - __path = None - __tid = None - __task_name = None - __branch_name = None - __user = None - __cwd = None - __rep = None - __diffs_dir = None - __annotation_file = None - __sync_date = None - __lfs = None - def __init__(self, db_git, tid, user): - self.__db_git = db_git - self.__url = db_git.url - self.__path = db_git.path - self.__tid = tid - self.__user = { + self._db_git = db_git + self._url = db_git.url + self._path = db_git.path + self._tid = tid + self._user = { "name": user.username, "email": user.email or "dummy@cvat.com" } - self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repos") - self.__diffs_dir = os.path.join(os.getcwd(), "data", str(tid), "repos_diffs") - self.__task_name = re.sub(r'[\\/*?:"<>|\s]', '_', Task.objects.get(pk = tid).name)[:100] - self.__branch_name = 'cvat_{}_{}'.format(tid, self.__task_name) - self.__annotation_file = os.path.join(self.__cwd, self.__path) - self.__sync_date = db_git.sync_date - self.__lfs = db_git.lfs + self._cwd = os.path.join(os.getcwd(), "data", str(tid), "repos") + self._diffs_dir = os.path.join(os.getcwd(), "data", str(tid), "repos_diffs_v2") + self._task_name = re.sub(r'[\\/*?:"<>|\s]', '_', Task.objects.get(pk = tid).name)[:100] + self._branch_name = 'cvat_{}_{}'.format(tid, self._task_name) + self._annotation_file = os.path.join(self._cwd, self._path) + self._sync_date = db_git.sync_date + self._lfs = db_git.lfs # Method parses an got URL. @@ -78,8 +66,8 @@ def _parse_url(self): http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)" ssh_pattern = "([a-zA-Z0-9._-]+)@([a-zA-Z0-9._-]+):([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)" - http_match = re.match(http_pattern, self.__url) - ssh_match = re.match(ssh_pattern, self.__url) + http_match = re.match(http_pattern, self._url) + ssh_match = re.match(ssh_pattern, self._url) user = "git" host = None @@ -106,44 +94,43 @@ def _parse_url(self): # Method creates the main branch if repostory doesn't have any branches def _create_master_branch(self): - if len(self.__rep.heads): + if len(self._rep.heads): raise Exception("Some heads already exists") - readme_md_name = os.path.join(self.__cwd, "README.md") + readme_md_name = os.path.join(self._cwd, "README.md") with open(readme_md_name, "w"): pass - self.__rep.index.add([readme_md_name]) - self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user["name"], timezone.now())) + self._rep.index.add([readme_md_name]) + self._rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self._user["name"], timezone.now())) - self.__rep.git.push("origin", "master") + self._rep.git.push("origin", "master") # Method creates task branch for repository from current master def _to_task_branch(self): # Remove user branch from local repository if it exists - if self.__branch_name not in list(map(lambda x: x.name, self.__rep.heads)): - self.__rep.create_head(self.__branch_name) + if self._branch_name not in list(map(lambda x: x.name, self._rep.heads)): + self._rep.create_head(self._branch_name) - self.__rep.head.reference = self.__rep.heads[self.__branch_name] + self._rep.head.reference = self._rep.heads[self._branch_name] # Method setups a config file for current user def _update_config(self): - slogger.task[self.__tid].info("User config initialization..") - with self.__rep.config_writer() as cw: + slogger.task[self._tid].info("User config initialization..") + with self._rep.config_writer() as cw: if not cw.has_section("user"): cw.add_section("user") - cw.set("user", "name", self.__user["name"]) - cw.set("user", "email", self.__user["email"]) + cw.set("user", "name", self._user["name"]) + cw.set("user", "email", self._user["email"]) cw.release() # Method initializes repos. It setup configuration, creates master branch if need and checkouts to task branch def _configurate(self): self._update_config() - if not len(self.__rep.heads): + if not len(self._rep.heads): self._create_master_branch() self._to_task_branch() - os.makedirs(self.__diffs_dir, exist_ok = True) - + os.makedirs(self._diffs_dir, exist_ok = True) def _ssh_url(self): @@ -153,12 +140,12 @@ def _ssh_url(self): # Method clones a remote repos to the local storage using SSH and initializes it def _clone(self): - os.makedirs(self.__cwd) + os.makedirs(self._cwd) ssh_url = self._ssh_url() # Cloning - slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url)) - self.__rep = git.Repo.clone_from(ssh_url, self.__cwd) + slogger.task[self._tid].info("Cloning remote repository from {}..".format(ssh_url)) + self._rep = git.Repo.clone_from(ssh_url, self._cwd) # Intitialization self._configurate() @@ -168,13 +155,13 @@ def _clone(self): # It restores state if any errors have occured # It useful if merge conflicts have occured during pull def _reclone(self): - if os.path.exists(self.__cwd): - if not os.path.isdir(self.__cwd): - os.remove(self.__cwd) + if os.path.exists(self._cwd): + if not os.path.isdir(self._cwd): + os.remove(self._cwd) else: # Rename current repository dir - tmp_repo = os.path.abspath(os.path.join(self.__cwd, "..", "tmp_repo")) - os.rename(self.__cwd, tmp_repo) + tmp_repo = os.path.abspath(os.path.join(self._cwd, "..", "tmp_repo")) + os.rename(self._cwd, tmp_repo) # Try clone repository try: @@ -182,9 +169,9 @@ def _reclone(self): shutil.rmtree(tmp_repo, True) except Exception as ex: # Restore state if any errors have occured - if os.path.isdir(self.__cwd): - shutil.rmtree(self.__cwd, True) - os.rename(tmp_repo, self.__cwd) + if os.path.isdir(self._cwd): + shutil.rmtree(self._cwd, True) + os.rename(tmp_repo, self._cwd) raise ex else: self._clone() @@ -192,14 +179,14 @@ def _reclone(self): # Method checkouts to master branch and pulls it from remote repos def _pull(self): - self.__rep.head.reference = self.__rep.heads["master"] + self._rep.head.reference = self._rep.heads["master"] try: - self.__rep.git.pull("origin", "master") + self._rep.git.pull("origin", "master") - if self.__branch_name in list(map(lambda x: x.name, self.__rep.heads)): - self.__rep.head.reference = self.__rep.heads["master"] - self.__rep.delete_head(self.__branch_name, force=True) - self.__rep.head.reset("HEAD", index=True, working_tree=True) + if self._branch_name in list(map(lambda x: x.name, self._rep.heads)): + self._rep.head.reference = self._rep.heads["master"] + self._rep.delete_head(self._branch_name, force=True) + self._rep.head.reset("HEAD", index=True, working_tree=True) self._to_task_branch() except git.exc.GitError: @@ -212,62 +199,42 @@ def _pull(self): def init_repos(self, wo_remote = False): try: # Try to use a local repos. It can throw GitError exception - self.__rep = git.Repo(self.__cwd) + self._rep = git.Repo(self._cwd) self._configurate() # Check if remote URL is actual - if self._ssh_url() != self.__rep.git.remote('get-url', '--all', 'origin'): - slogger.task[self.__tid].info("Local repository URL is obsolete.") + if self._ssh_url() != self._rep.git.remote('get-url', '--all', 'origin'): + slogger.task[self._tid].info("Local repository URL is obsolete.") # We need reinitialize repository if it's false raise git.exc.GitError("Actual and saved repository URLs aren't match") except git.exc.GitError: if wo_remote: raise Exception('Local repository is failed') - slogger.task[self.__tid].info("Local repository initialization..") - shutil.rmtree(self.__cwd, True) + slogger.task[self._tid].info("Local repository initialization..") + shutil.rmtree(self._cwd, True) self._clone() # Method prepares an annotation, merges diffs and pushes it to remote repository to user branch - def push(self, scheme, host, format, last_save): - - # Helpful function which merges diffs - def _accumulate(source, target, target_key): - if isinstance(source, dict): - if target_key is not None and target_key not in target: - target[target_key] = {} - - for key in source: - if target_key is not None: - _accumulate(source[key], target[target_key], key) - else: - _accumulate(source[key], target, key) - elif isinstance(source, int): - if source: - if target_key is not None and target_key not in target: - target[target_key] = 0 - target[target_key] += source - else: - raise Exception("Unhandled accumulate type: {}".format(type(source))) - + def push(self, scheme, host, db_task, last_save): # Update local repository self._pull() - os.makedirs(os.path.join(self.__cwd, os.path.dirname(self.__annotation_file)), exist_ok = True) + os.makedirs(os.path.join(self._cwd, os.path.dirname(self._annotation_file)), exist_ok = True) # Remove old annotation file if it exists - if os.path.exists(self.__annotation_file): - os.remove(self.__annotation_file) + if os.path.exists(self._annotation_file): + os.remove(self._annotation_file) # Initialize LFS if need - if self.__lfs: + if self._lfs: updated = False lfs_settings = ["*.xml\tfilter=lfs diff=lfs merge=lfs -text\n", "*.zip\tfilter=lfs diff=lfs merge=lfs -text\n"] - if not os.path.isfile(os.path.join(self.__cwd, ".gitattributes")): - with open(os.path.join(self.__cwd, ".gitattributes"), "w") as gitattributes: + if not os.path.isfile(os.path.join(self._cwd, ".gitattributes")): + with open(os.path.join(self._cwd, ".gitattributes"), "w") as gitattributes: gitattributes.writelines(lfs_settings) updated = True else: - with open(os.path.join(self.__cwd, ".gitattributes"), "r+") as gitattributes: + with open(os.path.join(self._cwd, ".gitattributes"), "r+") as gitattributes: lines = gitattributes.readlines() for setting in lfs_settings: if setting not in lines: @@ -278,51 +245,69 @@ def _accumulate(source, target, target_key): gitattributes.truncate() if updated: - self.__rep.git.add(['.gitattributes']) + self._rep.git.add(['.gitattributes']) # Dump an annotation - dump(self.__tid, format, scheme, host, OrderedDict()) - dump_name = Task.objects.get(pk = self.__tid).get_dump_path() + # TODO: Fix dump, query params + timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + dump_name = os.path.join(db_task.get_task_dirname(), + "git_annotation_{}.".format(timestamp) + "dump") + dump_task_data(self._tid, dump_name, scheme, host, {}) - ext = os.path.splitext(self.__path)[1] + ext = os.path.splitext(self._path)[1] if ext == '.zip': - subprocess.call('zip -j -r "{}" "{}"'.format(self.__annotation_file, dump_name), shell=True) + subprocess.call('zip -j -r "{}" "{}"'.format(self._annotation_file, dump_name), shell=True) elif ext == '.xml': - shutil.copyfile(dump_name, self.__annotation_file) + shutil.copyfile(dump_name, self._annotation_file) else: raise Exception("Got unknown annotation file type") - self.__rep.git.add(self.__annotation_file) + os.remove(dump_name) + self._rep.git.add(self._annotation_file) # Merge diffs summary_diff = {} - for diff_name in list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))): + for diff_name in list(map(lambda x: os.path.join(self._diffs_dir, x), os.listdir(self._diffs_dir))): with open(diff_name, 'r') as f: diff = json.loads(f.read()) - _accumulate(diff, summary_diff, None) + for key in diff: + if key not in summary_diff: + summary_diff[key] = 0 + summary_diff[key] += diff[key] + + message = "CVAT Annotation updated by {}. \n".format(self._user["name"]) + message += 'Task URL: {}://{}/dashboard?id={}\n'.format(scheme, host, db_task.id) + if db_task.bug_tracker: + message += 'Bug Tracker URL: {}\n'.format(db_task.bug_tracker) + message += "Created: {}, updated: {}, deleted: {}\n".format( + summary_diff["create"], + summary_diff["update"], + summary_diff["delete"] + ) + message += "Annotation time: {} hours".format(math.ceil((last_save - db_task.created_date).seconds / 3600)) - self.__rep.index.commit("CVAT Annotation updated by {}. Summary: {}".format(self.__user["name"], str(summary_diff))) - self.__rep.git.push("origin", self.__branch_name, "--force") + self._rep.index.commit(message) + self._rep.git.push("origin", self._branch_name, "--force") - shutil.rmtree(self.__diffs_dir, True) + shutil.rmtree(self._diffs_dir, True) # Method checks status of repository annotation def remote_status(self, last_save): # Check repository exists and archive exists - if not os.path.isfile(self.__annotation_file) or last_save != self.__sync_date: + if not os.path.isfile(self._annotation_file) or last_save != self._sync_date: return GitStatusChoice.NON_SYNCED else: - self.__rep.git.update_ref('-d', 'refs/remotes/origin/{}'.format(self.__branch_name)) - self.__rep.git.remote('-v', 'update') + self._rep.git.update_ref('-d', 'refs/remotes/origin/{}'.format(self._branch_name)) + self._rep.git.remote('-v', 'update') - last_hash = self.__rep.git.show_ref('refs/heads/{}'.format(self.__branch_name), '--hash') - merge_base_hash = self.__rep.merge_base('refs/remotes/origin/master', self.__branch_name)[0].hexsha + last_hash = self._rep.git.show_ref('refs/heads/{}'.format(self._branch_name), '--hash') + merge_base_hash = self._rep.merge_base('refs/remotes/origin/master', self._branch_name)[0].hexsha if last_hash == merge_base_hash: return GitStatusChoice.MERGED else: try: - self.__rep.git.show_ref('refs/remotes/origin/{}'.format(self.__branch_name), '--hash') + self._rep.git.show_ref('refs/remotes/origin/{}'.format(self._branch_name), '--hash') return GitStatusChoice.SYNCED except git.exc.GitCommandError: # Remote branch has been deleted w/o merge @@ -374,7 +359,7 @@ def push(tid, user, scheme, host): try: _git = Git(db_git, tid, user) _git.init_repos() - _git.push(scheme, host, FORMAT_XML, db_task.updated_date) + _git.push(scheme, host, db_task, db_task.updated_date) # Update timestamp db_git.sync_date = db_task.updated_date @@ -433,33 +418,41 @@ def update_states(): slogger.glob("Exception occured during a status updating for db_git with tid: {}".format(db_git.task_id)) @transaction.atomic -def _onsave(jid, data): +def _onsave(jid, data, action): db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task try: db_git = GitData.objects.select_for_update().get(pk = db_task.id) diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repos_diffs") - os.makedirs(diff_dir, exist_ok = True) + diff_dir_v2 = os.path.join(os.getcwd(), "data", str(db_task.id), "repos_diffs_v2") - updated = sum([ len(data["update"][key]) for key in data["update"] ]) - deleted = sum([ len(data["delete"][key]) for key in data["delete"] ]) - created = sum([ len(data["create"][key]) for key in data["create"] ]) + summary = { + "update": 0, + "create": 0, + "delete": 0 + } + if os.path.isdir(diff_dir) and not os.path.isdir(diff_dir_v2): + diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + for diff_file in diff_files: + diff_file = open(diff_file, 'r') + diff = json.loads(diff_file.read()) - if updated or deleted or created: - diff = { - "update": {key: len(data["update"][key]) for key in data["update"].keys()}, - "delete": {key: len(data["delete"][key]) for key in data["delete"].keys()}, - "create": {key: len(data["create"][key]) for key in data["create"].keys()} - } + for action_key in diff: + summary[action_key] += sum([diff[action_key][key] for key in diff[action_key]]) - diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir))) + os.makedirs(diff_dir_v2, exist_ok = True) + + summary[action] += sum([len(data[key]) for key in ['shapes', 'tracks', 'tags']]) + + if summary["update"] or summary["create"] or summary["delete"]: + diff_files = list(map(lambda x: os.path.join(diff_dir_v2, x), os.listdir(diff_dir_v2))) last_num = 0 for f in diff_files: number = os.path.splitext(os.path.basename(f))[0] number = int(number) if number.isdigit() else last_num last_num = max(last_num, number) - with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f: - f.write(json.dumps(diff)) + with open(os.path.join(diff_dir_v2, "{}.diff".format(last_num + 1)), 'w') as f: + f.write(json.dumps(summary)) db_git.status = GitStatusChoice.NON_SYNCED db_git.save() @@ -478,5 +471,7 @@ def _ondump(tid, data_format, scheme, host, plugin_meta_data): except GitData.DoesNotExist: pass -add_plugin("save_job", _onsave, "after", exc_ok = False) -add_plugin("_dump", _ondump, "before", exc_ok = False) +add_plugin("patch_job_data", _onsave, "after", exc_ok = False) + +# TODO: Append git repository into dump file +# add_plugin("_dump", _ondump, "before", exc_ok = False)