diff --git a/cvat/apps/dashboard/static/dashboard/js/dashboard.js b/cvat/apps/dashboard/static/dashboard/js/dashboard.js
index 4a8986668ca4..a6cc4f393a54 100644
--- a/cvat/apps/dashboard/static/dashboard/js/dashboard.js
+++ b/cvat/apps/dashboard/static/dashboard/js/dashboard.js
@@ -542,11 +542,15 @@ function uploadAnnotationRequest() {
return;
}
+ const exportData = createExportContainer();
+ exportData.create = parsed;
+ exportData.pre_erase = true;
+
let asyncSave = function() {
$.ajax({
url: '/save/annotation/task/' + window.cvat.dashboard.taskID,
type: 'POST',
- data: JSON.stringify(parsed),
+ data: JSON.stringify(exportData),
contentType: 'application/json',
success: function() {
let message = 'Annotation successfully uploaded';
diff --git a/cvat/apps/engine/annotation.py b/cvat/apps/engine/annotation.py
index 8cd9ebe64cee..4488e2459d12 100644
--- a/cvat/apps/engine/annotation.py
+++ b/cvat/apps/engine/annotation.py
@@ -11,7 +11,6 @@
from scipy.optimize import linear_sum_assignment
from collections import OrderedDict
from distutils.util import strtobool
-from xml.dom import minidom
from xml.sax.saxutils import XMLGenerator
from abc import ABCMeta, abstractmethod
from PIL import Image
@@ -77,9 +76,16 @@ def save_job(jid, data):
Save new annotations for the job.
"""
db_job = models.Job.objects.select_for_update().get(id=jid)
+
annotation = _AnnotationForJob(db_job)
- annotation.init_from_client(data)
- annotation.save_to_db()
+ annotation.validate_data_from_client(data)
+ if data['pre_erase']:
+ annotation.delete_objs_from_db()
+
+ for action in ['create', 'update', 'delete']:
+ annotation.init_from_client(data[action])
+ annotation.save_to_db(action)
+
db_job.segment.task.updated_date = timezone.now()
db_job.segment.task.save()
@@ -97,16 +103,20 @@ def save_task(tid, data):
jid = segment.job_set.first().id
start = segment.start_frame
stop = segment.stop_frame
- splitted_data[jid] = {
- "boxes": list(filter(lambda x: start <= int(x['frame']) <= stop, data['boxes'])),
- "polygons": list(filter(lambda x: start <= int(x['frame']) <= stop, data['polygons'])),
- "polylines": list(filter(lambda x: start <= int(x['frame']) <= stop, data['polylines'])),
- "points": list(filter(lambda x: start <= int(x['frame']) <= stop, data['points'])),
- "box_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data['box_paths'])),
- "polygon_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data['polygon_paths'])),
- "polyline_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data['polyline_paths'])),
- "points_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data['points_paths'])),
- }
+ splitted_data[jid] = {}
+ for action in ['create', 'update', 'delete']:
+ splitted_data[jid][action] = {
+ "boxes": list(filter(lambda x: start <= int(x['frame']) <= stop, data[action]['boxes'])),
+ "polygons": list(filter(lambda x: start <= int(x['frame']) <= stop, data[action]['polygons'])),
+ "polylines": list(filter(lambda x: start <= int(x['frame']) <= stop, data[action]['polylines'])),
+ "points": list(filter(lambda x: start <= int(x['frame']) <= stop, data[action]['points'])),
+ "box_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data[action]['box_paths'])),
+ "polygon_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data[action]['polygon_paths'])),
+ "polyline_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data[action]['polyline_paths'])),
+ "points_paths": list(filter(lambda x: len(list(filter(lambda y: (start <= int(y['frame']) <= stop) and (not y['outside']), x['shapes']))), data[action]['points_paths'])),
+ }
+
+ splitted_data[jid]['pre_erase'] = data['pre_erase']
for jid, _data in splitted_data.items():
save_job(jid, _data)
@@ -133,13 +143,14 @@ def __init__(self, db_attr, value):
self.value = str(value)
class _BoundingBox:
- def __init__(self, x0, y0, x1, y1, frame, occluded, z_order, attributes=None):
+ def __init__(self, x0, y0, x1, y1, frame, occluded, z_order, client_id=None, attributes=None):
self.xtl = x0
self.ytl = y0
self.xbr = x1
self.ybr = y1
self.occluded = occluded
self.z_order = z_order
+ self.client_id = client_id
self.frame = frame
self.attributes = attributes if attributes else []
@@ -156,14 +167,14 @@ def add_attribute(self, attr):
self.attributes.append(attr)
class _LabeledBox(_BoundingBox):
- def __init__(self, label, x0, y0, x1, y1, frame, group_id, occluded, z_order, attributes=None):
- super().__init__(x0, y0, x1, y1, frame, occluded, z_order, attributes)
+ def __init__(self, label, x0, y0, x1, y1, frame, group_id, occluded, z_order, client_id=None, attributes=None):
+ super().__init__(x0, y0, x1, y1, frame, occluded, z_order, client_id, attributes)
self.label = label
self.group_id = group_id
class _TrackedBox(_BoundingBox):
def __init__(self, x0, y0, x1, y1, frame, occluded, z_order, outside, attributes=None):
- super().__init__(x0, y0, x1, y1, frame, occluded, z_order, attributes)
+ super().__init__(x0, y0, x1, y1, frame, occluded, z_order, None, attributes)
self.outside = outside
class _InterpolatedBox(_TrackedBox):
@@ -172,25 +183,26 @@ def __init__(self, x0, y0, x1, y1, frame, occluded, z_order, outside, keyframe,
self.keyframe = keyframe
class _PolyShape:
- def __init__(self, points, frame, occluded, z_order, attributes=None):
+ def __init__(self, points, frame, occluded, z_order, client_id=None, attributes=None):
self.points = points
+ self.frame = frame
self.occluded = occluded
self.z_order = z_order
- self.frame = frame
+ self.client_id=client_id
self.attributes = attributes if attributes else []
def add_attribute(self, attr):
self.attributes.append(attr)
class _LabeledPolyShape(_PolyShape):
- def __init__(self, label, points, frame, group_id, occluded, z_order, attributes=None):
- super().__init__(points, frame, occluded, z_order, attributes)
+ def __init__(self, label, points, frame, group_id, occluded, z_order, client_id=None, attributes=None):
+ super().__init__(points, frame, occluded, z_order, client_id, attributes)
self.label = label
self.group_id = group_id
class _TrackedPolyShape(_PolyShape):
def __init__(self, points, frame, occluded, z_order, outside, attributes=None):
- super().__init__(points, frame, occluded, z_order, attributes)
+ super().__init__(points, frame, occluded, z_order, None, attributes)
self.outside = outside
class _InterpolatedPolyShape(_TrackedPolyShape):
@@ -199,12 +211,13 @@ def __init__(self, points, frame, occluded, z_order, outside, keyframe, attribut
self.keyframe = keyframe
class _BoxPath:
- def __init__(self, label, start_frame, stop_frame, group_id, boxes=None, attributes=None):
+ def __init__(self, label, start_frame, stop_frame, group_id, boxes=None, client_id=None, attributes=None):
self.label = label
self.frame = start_frame
- self.group_id = group_id
self.stop_frame = stop_frame
+ self.group_id = group_id
self.boxes = boxes if boxes else []
+ self.client_id = client_id
self.attributes = attributes if attributes else []
self._interpolated_boxes = []
assert not self.boxes or self.boxes[-1].frame <= self.stop_frame
@@ -273,12 +286,13 @@ def add_attribute(self, attr):
self.attributes.append(attr)
class _PolyPath:
- def __init__(self, label, start_frame, stop_frame, group_id, shapes=None, attributes=None):
+ def __init__(self, label, start_frame, stop_frame, group_id, shapes=None, client_id=None, attributes=None):
self.label = label
self.frame = start_frame
- self.group_id = group_id
self.stop_frame = stop_frame
+ self.group_id = group_id
self.shapes = shapes if shapes else []
+ self.client_id = client_id
self.attributes = attributes if attributes else []
self._interpolated_shapes = [] # ???
@@ -333,36 +347,82 @@ def reset(self):
self.points = []
self.points_paths = []
+ def get_max_client_id(self):
+ max_client_id = -1
+
+ def extract_client_id(shape):
+ return shape.client_id
+
+ if self.boxes:
+ max_client_id = max(max_client_id, (max(self.boxes, key=extract_client_id)).client_id)
+ if self.box_paths:
+ max_client_id = max(max_client_id, (max(self.box_paths, key=extract_client_id)).client_id)
+ if self.polygons:
+ max_client_id = max(max_client_id, (max(self.polygons, key=extract_client_id)).client_id)
+ if self.polygon_paths:
+ max_client_id = max(max_client_id, (max(self.polygon_paths, key=extract_client_id)).client_id)
+ if self.polylines:
+ max_client_id = max(max_client_id, (max(self.polylines, key=extract_client_id)).client_id)
+ if self.polyline_paths:
+ max_client_id = max(max_client_id, (max(self.polyline_paths, key=extract_client_id)).client_id)
+ if self.points:
+ max_client_id = max(max_client_id, (max(self.points, key=extract_client_id)).client_id)
+ if self.points_paths:
+ max_client_id = max(max_client_id, (max(self.points_paths, key=extract_client_id)).client_id)
+
+ return max_client_id
+
# Functions below used by dump functionality
- def to_boxes(self):
+ def to_boxes(self, start_client_id):
boxes = []
for path in self.box_paths:
for box in path.get_interpolated_boxes():
if not box.outside:
- box = _LabeledBox(path.label, box.xtl, box.ytl, box.xbr, box.ybr,
- box.frame, path.group_id, box.occluded, box.z_order, box.attributes + path.attributes)
+ box = _LabeledBox(
+ label=path.label,
+ x0=box.xtl, y0=box.ytl, x1=box.xbr, y1=box.ybr,
+ frame=box.frame,
+ group_id=path.group_id,
+ occluded=box.occluded,
+ z_order=box.z_order,
+ client_id=start_client_id,
+ attributes=box.attributes + path.attributes,
+ )
boxes.append(box)
+ start_client_id += 1
- return self.boxes + boxes
+ return self.boxes + boxes, start_client_id
- def _to_poly_shapes(self, iter_attr_name):
+ def _to_poly_shapes(self, iter_attr_name, start_client_id):
shapes = []
for path in getattr(self, iter_attr_name):
for shape in path.get_interpolated_shapes():
if not shape.outside:
- shape = _LabeledPolyShape(path.label, shape.points, shape.frame, path.group_id,
- shape.occluded, shape.z_order, shape.attributes + path.attributes)
+ shape = _LabeledPolyShape(
+ label=path.label,
+ points=shape.points,
+ frame=shape.frame,
+ group_id=path.group_id,
+ occluded=shape.occluded,
+ z_order=shape.z_order,
+ client_id=start_client_id,
+ attributes=shape.attributes + path.attributes,
+ )
shapes.append(shape)
- return shapes
+ start_client_id += 1
+ return shapes, start_client_id
- def to_polygons(self):
- return self._to_poly_shapes('polygon_paths') + self.polygons
+ def to_polygons(self, start_client_id):
+ polygons, client_id = self._to_poly_shapes('polygon_paths', start_client_id)
+ return polygons + self.polygons, client_id
- def to_polylines(self):
- return self._to_poly_shapes('polyline_paths') + self.polylines
+ def to_polylines(self, start_client_id):
+ polylines, client_id = self._to_poly_shapes('polyline_paths', start_client_id)
+ return polylines + self.polylines, client_id
- def to_points(self):
- return self._to_poly_shapes('points_paths') + self.points
+ def to_points(self, start_client_id):
+ points, client_id = self._to_poly_shapes('points_paths', start_client_id)
+ return points + self.points, client_id
def to_box_paths(self):
paths = []
@@ -372,7 +432,15 @@ def to_box_paths(self):
box1 = copy.copy(box0)
box1.outside = True
box1.frame += 1
- path = _BoxPath(box.label, box.frame, box.frame + 1, box.group_id, [box0, box1], box.attributes)
+ path = _BoxPath(
+ label=box.label,
+ start_frame=box.frame,
+ stop_frame=box.frame + 1,
+ group_id=box.group_id,
+ boxes=[box0, box1],
+ attributes=box.attributes,
+ client_id=box.client_id,
+ )
paths.append(path)
return self.box_paths + paths
@@ -385,7 +453,15 @@ def _to_poly_paths(self, iter_attr_name):
shape1 = copy.copy(shape0)
shape1.outside = True
shape1.frame += 1
- path = _PolyPath(shape.label, shape.frame, shape.frame + 1, shape.group_id, [shape0, shape1], shape.attributes)
+ path = _PolyPath(
+ label=shape.label,
+ start_frame=shape.frame,
+ stop_frame=shape.frame + 1,
+ group_id=shape.group_id,
+ shapes=[shape0, shape1],
+ client_id=shape.client_id,
+ attributes=shape.attributes,
+ )
paths.append(path)
return paths
@@ -450,11 +526,16 @@ class dotdict(OrderedDict):
return list(merged_rows.values())
+ @staticmethod
+ def _clamp(value, min_value, max_value):
+ return max(min(value, max_value), min_value)
+
def _clamp_box(self, xtl, ytl, xbr, ybr, im_size):
- xtl = max(min(xtl, im_size['width']), 0)
- ytl = max(min(ytl, im_size['height']), 0)
- xbr = max(min(xbr, im_size['width']), 0)
- ybr = max(min(ybr, im_size['height']), 0)
+ xtl = self._clamp(xtl, 0, im_size['width'])
+ xbr = self._clamp(xbr, 0, im_size['width'])
+ ytl = self._clamp(ytl, 0, im_size['height'])
+ ybr = self._clamp(ybr, 0, im_size['height'])
+
return xtl, ytl, xbr, ybr
def _clamp_poly(self, points, im_size):
@@ -463,16 +544,17 @@ def _clamp_poly(self, points, im_size):
for p in points:
p = p.split(',')
verified.append('{},{}'.format(
- max(min(float(p[0]), im_size['width']), 0),
- max(min(float(p[1]), im_size['height']), 0)
+ self._clamp(float(p[0]), 0, im_size['width']),
+ self._clamp(float(p[1]), 0, im_size['height'])
))
+
return ' '.join(verified)
def init_from_db(self):
def get_values(shape_type):
if shape_type == 'polygons':
return [
- ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order',
+ ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order', 'client_id',
'labeledpolygonattributeval__value', 'labeledpolygonattributeval__spec_id',
'labeledpolygonattributeval__id'), {
'attributes': [
@@ -484,7 +566,7 @@ def get_values(shape_type):
]
elif shape_type == 'polylines':
return [
- ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order',
+ ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order', 'client_id',
'labeledpolylineattributeval__value', 'labeledpolylineattributeval__spec_id',
'labeledpolylineattributeval__id'), {
'attributes': [
@@ -496,7 +578,7 @@ def get_values(shape_type):
]
elif shape_type == 'boxes':
return [
- ('id', 'frame', 'xtl', 'ytl', 'xbr', 'ybr', 'label_id', 'group_id', 'occluded', 'z_order',
+ ('id', 'frame', 'xtl', 'ytl', 'xbr', 'ybr', 'label_id', 'group_id', 'occluded', 'z_order', 'client_id',
'labeledboxattributeval__value', 'labeledboxattributeval__spec_id',
'labeledboxattributeval__id'), {
'attributes': [
@@ -508,7 +590,7 @@ def get_values(shape_type):
]
elif shape_type == 'points':
return [
- ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order',
+ ('id', 'frame', 'points', 'label_id', 'group_id', 'occluded', 'z_order', 'client_id',
'labeledpointsattributeval__value', 'labeledpointsattributeval__spec_id',
'labeledpointsattributeval__id'), {
'attributes': [
@@ -528,11 +610,24 @@ def get_values(shape_type):
for db_shape in db_shapes:
label = _Label(self.db_labels[db_shape.label_id])
if shape_type == 'boxes':
- shape = _LabeledBox(label, db_shape.xtl, db_shape.ytl, db_shape.xbr, db_shape.ybr,
- db_shape.frame, db_shape.group_id, db_shape.occluded, db_shape.z_order)
+ shape = _LabeledBox(label=label,
+ x0=db_shape.xtl, y0=db_shape.ytl, x1=db_shape.xbr, y1=db_shape.ybr,
+ frame=db_shape.frame,
+ group_id=db_shape.group_id,
+ occluded=db_shape.occluded,
+ z_order=db_shape.z_order,
+ client_id=db_shape.client_id,
+ )
else:
- shape = _LabeledPolyShape(label, db_shape.points, db_shape.frame,
- db_shape.group_id, db_shape.occluded, db_shape.z_order)
+ shape = _LabeledPolyShape(
+ label=label,
+ points=db_shape.points,
+ frame=db_shape.frame,
+ group_id=db_shape.group_id,
+ occluded=db_shape.occluded,
+ z_order=db_shape.z_order,
+ client_id=db_shape.client_id,
+ )
for db_attr in db_shape.attributes:
if db_attr.id != None:
spec = self.db_attributes[db_attr.spec_id]
@@ -540,8 +635,6 @@ def get_values(shape_type):
shape.add_attribute(attr)
getattr(self, shape_type).append(shape)
-
-
db_paths = self.db_job.objectpath_set
for shape in ['trackedpoints_set', 'trackedbox_set', 'trackedpolyline_set', 'trackedpolygon_set']:
db_paths.prefetch_related(shape)
@@ -549,7 +642,7 @@ def get_values(shape_type):
'trackedpolygon_set__trackedpolygonattributeval_set', 'trackedpolyline_set__trackedpolylineattributeval_set']:
db_paths.prefetch_related(shape_attr)
db_paths.prefetch_related('objectpathattributeval_set')
- db_paths = list (db_paths.values('id', 'frame', 'group_id', 'shapes', 'objectpathattributeval__spec_id',
+ db_paths = list (db_paths.values('id', 'frame', 'group_id', 'shapes', 'client_id', 'objectpathattributeval__spec_id',
'objectpathattributeval__id', 'objectpathattributeval__value',
'trackedbox', 'trackedpolygon', 'trackedpolyline', 'trackedpoints',
'trackedbox__id', 'label_id', 'trackedbox__xtl', 'trackedbox__ytl',
@@ -667,7 +760,13 @@ def get_values(shape_type):
for db_shape in db_path.shapes:
db_shape.attributes = list(set(db_shape.attributes))
label = _Label(self.db_labels[db_path.label_id])
- path = _BoxPath(label, db_path.frame, self.stop_frame, db_path.group_id)
+ path = _BoxPath(
+ label=label,
+ start_frame=db_path.frame,
+ stop_frame=self.stop_frame,
+ group_id=db_path.group_id,
+ client_id=db_path.client_id,
+ )
for db_attr in db_path.attributes:
spec = self.db_attributes[db_attr.spec_id]
attr = _Attribute(spec, db_attr.value)
@@ -675,8 +774,13 @@ def get_values(shape_type):
frame = -1
for db_shape in db_path.shapes:
- box = _TrackedBox(db_shape.xtl, db_shape.ytl, db_shape.xbr, db_shape.ybr,
- db_shape.frame, db_shape.occluded, db_shape.z_order, db_shape.outside)
+ box = _TrackedBox(
+ x0=db_shape.xtl, y0=db_shape.ytl, x1=db_shape.xbr, y1=db_shape.ybr,
+ frame=db_shape.frame,
+ occluded=db_shape.occluded,
+ z_order=db_shape.z_order,
+ outside=db_shape.outside,
+ )
assert box.frame > frame
frame = box.frame
@@ -695,7 +799,13 @@ def get_values(shape_type):
for db_shape in db_path.shapes:
db_shape.attributes = list(set(db_shape.attributes))
label = _Label(self.db_labels[db_path.label_id])
- path = _PolyPath(label, db_path.frame, self.stop_frame, db_path.group_id)
+ path = _PolyPath(
+ label=label,
+ start_frame=db_path.frame,
+ stop_frame= self.stop_frame,
+ group_id=db_path.group_id,
+ client_id=db_path.id,
+ )
for db_attr in db_path.attributes:
spec = self.db_attributes[db_attr.spec_id]
attr = _Attribute(spec, db_attr.value)
@@ -703,7 +813,13 @@ def get_values(shape_type):
frame = -1
for db_shape in db_path.shapes:
- shape = _TrackedPolyShape(db_shape.points, db_shape.frame, db_shape.occluded, db_shape.z_order, db_shape.outside)
+ shape = _TrackedPolyShape(
+ points=db_shape.points,
+ frame=db_shape.frame,
+ occluded=db_shape.occluded,
+ z_order=db_shape.z_order,
+ outside=db_shape.outside,
+ )
assert shape.frame > frame
frame = shape.frame
@@ -731,8 +847,16 @@ def init_from_client(self, data):
xtl, ytl, xbr, ybr = self._clamp_box(float(box['xtl']), float(box['ytl']),
float(box['xbr']), float(box['ybr']),
image_meta['original_size'][frame_idx])
- labeled_box = _LabeledBox(label, xtl, ytl, xbr, ybr, int(box['frame']),
- int(box['group_id']), strtobool(str(box['occluded'])), int(box['z_order']))
+
+ labeled_box = _LabeledBox(
+ label=label,
+ x0=xtl, y0=ytl, x1=xbr, y1=ybr,
+ frame=int(box['frame']),
+ group_id=int(box['group_id']),
+ occluded=strtobool(str(box['occluded'])),
+ z_order=int(box['z_order']),
+ client_id=int(box['client_id']),
+ )
for attr in box['attributes']:
spec = self.db_attributes[int(attr['id'])]
@@ -747,8 +871,15 @@ def init_from_client(self, data):
frame_idx = int(poly_shape['frame']) if db_task.mode == 'annotation' else 0
points = self._clamp_poly(poly_shape['points'], image_meta['original_size'][frame_idx])
- labeled_poly_shape = _LabeledPolyShape(label, points, int(poly_shape['frame']),
- int(poly_shape['group_id']), poly_shape['occluded'], int(poly_shape['z_order']))
+ labeled_poly_shape = _LabeledPolyShape(
+ label=label,
+ points=points,
+ frame=int(poly_shape['frame']),
+ group_id=int(poly_shape['group_id']),
+ occluded=poly_shape['occluded'],
+ z_order=int(poly_shape['z_order']),
+ client_id=int(poly_shape['client_id']),
+ )
for attr in poly_shape['attributes']:
spec = self.db_attributes[int(attr['id'])]
@@ -781,9 +912,14 @@ def init_from_client(self, data):
frame_idx = int(box['frame']) if db_task.mode == 'annotation' else 0
xtl, ytl, xbr, ybr = self._clamp_box(float(box['xtl']), float(box['ytl']),
float(box['xbr']), float(box['ybr']), image_meta['original_size'][frame_idx])
- tracked_box = _TrackedBox(xtl, ytl, xbr, ybr, int(box['frame']), strtobool(str(box['occluded'])),
- int(box['z_order']), strtobool(str(box['outside'])))
- assert tracked_box.frame > frame
+ tracked_box = _TrackedBox(
+ x0=xtl, y0=ytl, x1=xbr, y1=ybr,
+ frame=int(box['frame']),
+ occluded=strtobool(str(box['occluded'])),
+ z_order=int(box['z_order']),
+ outside=strtobool(str(box['outside'])),
+ )
+ assert tracked_box.frame > frame
frame = tracked_box.frame
for attr in box['attributes']:
@@ -805,8 +941,14 @@ def init_from_client(self, data):
attributes.append(attr)
assert frame <= self.stop_frame
- box_path = _BoxPath(label, min(list(map(lambda box: box.frame, boxes))), self.stop_frame,
- int(path['group_id']), boxes, attributes)
+ box_path = _BoxPath(label=label,
+ start_frame=min(list(map(lambda box: box.frame, boxes))),
+ stop_frame=self.stop_frame,
+ group_id=int(path['group_id']),
+ boxes=boxes,
+ client_id=int(path['client_id']),
+ attributes=attributes,
+ )
self.box_paths.append(box_path)
for poly_path_type in ['points_paths', 'polygon_paths', 'polyline_paths']:
@@ -833,8 +975,13 @@ def init_from_client(self, data):
if int(poly_shape['frame']) <= self.stop_frame and int(poly_shape['frame']) >= self.start_frame:
frame_idx = int(poly_shape['frame']) if db_task.mode == 'annotation' else 0
points = self._clamp_poly(poly_shape['points'], image_meta['original_size'][frame_idx])
- tracked_poly_shape = _TrackedPolyShape(points, int(poly_shape['frame']), strtobool(str(poly_shape['occluded'])),
- int(poly_shape['z_order']), strtobool(str(poly_shape['outside'])))
+ tracked_poly_shape = _TrackedPolyShape(
+ points=points,
+ frame=int(poly_shape['frame']),
+ occluded=strtobool(str(poly_shape['occluded'])),
+ z_order=int(poly_shape['z_order']),
+ outside=strtobool(str(poly_shape['outside'])),
+ )
assert tracked_poly_shape.frame > frame
frame = tracked_poly_shape.frame
@@ -856,8 +1003,15 @@ def init_from_client(self, data):
attr = _Attribute(spec, str(attr['value']))
attributes.append(attr)
- poly_path = _PolyPath(label, min(list(map(lambda shape: shape.frame, poly_shapes))), self.stop_frame + 1,
- int(path['group_id']), poly_shapes, attributes)
+ poly_path = _PolyPath(
+ label=label,
+ start_frame=min(list(map(lambda shape: shape.frame, poly_shapes))),
+ stop_frame=self.stop_frame + 1,
+ group_id=int(path['group_id']),
+ shapes=poly_shapes,
+ client_id=int(path['client_id']),
+ attributes=attributes,
+ )
getattr(self, poly_path_type).append(poly_path)
@@ -898,7 +1052,12 @@ def _get_shape_attr_class(self, shape_type):
return models.TrackedPointsAttributeVal
def _save_paths_to_db(self):
- self.db_job.objectpath_set.all().delete()
+ saved_path_ids = list(self.db_job.objectpath_set.values_list('id', 'client_id'))
+ saved_db_ids = []
+ saved_client_ids = []
+ for db_id, client_id in saved_path_ids:
+ saved_db_ids.append(db_id)
+ saved_client_ids.append(client_id)
for shape_type in ['polygon_paths', 'polyline_paths', 'points_paths', 'box_paths']:
db_paths = []
@@ -906,12 +1065,17 @@ def _save_paths_to_db(self):
db_shapes = []
db_shape_attrvals = []
- for path in getattr(self, shape_type):
+ shapes = getattr(self, shape_type)
+ for path in shapes:
+ if path.client_id in saved_client_ids:
+ raise Exception('Trying to create new shape with existing client_id {}'.format(path.client_id))
+
db_path = models.ObjectPath()
db_path.job = self.db_job
db_path.label = self.db_labels[path.label.id]
db_path.frame = path.frame
db_path.group_id = path.group_id
+ db_path.client_id = path.client_id
if shape_type == 'polygon_paths':
db_path.shapes = 'polygons'
elif shape_type == 'polyline_paths':
@@ -929,8 +1093,8 @@ def _save_paths_to_db(self):
db_attrval.value = attr.value
db_path_attrvals.append(db_attrval)
- shapes = path.boxes if hasattr(path, 'boxes') else path.shapes
- for shape in shapes:
+ path_shapes = path.boxes if hasattr(path, 'boxes') else path.shapes
+ for shape in path_shapes:
db_shape = self._get_shape_class(shape_type)()
db_shape.track_id = len(db_paths)
if shape_type == 'box_paths':
@@ -962,6 +1126,7 @@ def _save_paths_to_db(self):
db_shapes.append(db_shape)
db_paths.append(db_path)
+
db_paths = models.ObjectPath.objects.bulk_create(db_paths)
if db_paths and db_paths[0].id == None:
@@ -970,13 +1135,13 @@ def _save_paths_to_db(self):
# for Postgres bulk_create will return objects with ids even ids
# are auto incremented. Thus we will not be inside the 'if'.
if shape_type == 'polygon_paths':
- db_paths = list(self.db_job.objectpath_set.filter(shapes="polygons"))
+ db_paths = list(self.db_job.objectpath_set.exclude(id__in=saved_db_ids))
elif shape_type == 'polyline_paths':
- db_paths = list(self.db_job.objectpath_set.filter(shapes="polylines"))
+ db_paths = list(self.db_job.objectpath_set.exclude(id__in=saved_db_ids))
elif shape_type == 'box_paths':
- db_paths = list(self.db_job.objectpath_set.filter(shapes="boxes"))
+ db_paths = list(self.db_job.objectpath_set.exclude(id__in=saved_db_ids))
elif shape_type == 'points_paths':
- db_paths = list(self.db_job.objectpath_set.filter(shapes="points"))
+ db_paths = list(self.db_job.objectpath_set.exclude(id__in=saved_db_ids))
for db_attrval in db_path_attrvals:
db_attrval.track_id = db_paths[db_attrval.track_id].id
@@ -985,6 +1150,7 @@ def _save_paths_to_db(self):
for db_shape in db_shapes:
db_shape.track_id = db_paths[db_shape.track_id].id
+ db_shapes_ids = list(self._get_shape_class(shape_type).objects.filter(track__job_id=self.db_job.id).values_list('id', flat=True))
db_shapes = self._get_shape_class(shape_type).objects.bulk_create(db_shapes)
if db_shapes and db_shapes[0].id == None:
@@ -992,7 +1158,7 @@ def _save_paths_to_db(self):
# but it definetely doesn't work for Postgres. Need to say that
# for Postgres bulk_create will return objects with ids even ids
# are auto incremented. Thus we will not be inside the 'if'.
- db_shapes = list(self._get_shape_class(shape_type).objects.filter(track__job_id=self.db_job.id))
+ db_shapes = list(self._get_shape_class(shape_type).objects.exclude(id__in=db_shapes_ids).filter(track__job_id=self.db_job.id))
for db_attrval in db_shape_attrvals:
if shape_type == 'polygon_paths':
@@ -1021,15 +1187,27 @@ def _save_shapes_to_db(self):
db_attrvals = []
for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
- self._get_shape_set(shape_type).all().delete()
db_shapes = []
db_attrvals = []
- for shape in getattr(self, shape_type):
+ saved_shapes_ids = list(self._get_shape_class(shape_type).objects.filter(job_id=self.db_job.id).values_list('id', 'client_id'))
+ saved_client_ids = []
+ saved_db_ids = []
+
+ for db_id, client_id in saved_shapes_ids:
+ saved_db_ids.append(db_id)
+ saved_client_ids.append(client_id)
+
+ shapes = getattr(self, shape_type)
+ for shape in shapes:
+ if shape.client_id in saved_client_ids:
+ raise Exception('Trying to create new shape with existing client_id {}'.format(shape.client_id))
+
db_shape = self._get_shape_class(shape_type)()
db_shape.job = self.db_job
db_shape.label = self.db_labels[shape.label.id]
db_shape.group_id = shape.group_id
+ db_shape.client_id = shape.client_id
if shape_type == 'boxes':
db_shape.xtl = shape.xtl
db_shape.ytl = shape.ytl
@@ -1065,7 +1243,8 @@ def _save_shapes_to_db(self):
# but it definetely doesn't work for Postgres. Need to say that
# for Postgres bulk_create will return objects with ids even ids
# are auto incremented. Thus we will not be inside the 'if'.
- db_shapes = list(self._get_shape_set(shape_type).all())
+ db_shapes = list(self._get_shape_set(shape_type).exclude(id__in=saved_db_ids))
+
for db_attrval in db_attrvals:
if shape_type == 'polygons':
@@ -1079,10 +1258,53 @@ def _save_shapes_to_db(self):
self._get_shape_attr_class(shape_type).objects.bulk_create(db_attrvals)
- def save_to_db(self):
- self._save_shapes_to_db()
+ def _update_shapes_in_db(self):
+ self._delete_paths_from_db()
self._save_paths_to_db()
+ def _update_paths_in_db(self):
+ self._delete_shapes_from_db()
+ self._save_shapes_to_db()
+
+ def _delete_shapes_from_db(self):
+ for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
+ client_ids_to_delete = list(shape.client_id for shape in getattr(self, shape_type))
+ deleted = self._get_shape_set(shape_type).filter(client_id__in=client_ids_to_delete).delete()
+ class_name = 'engine.{}'.format(self._get_shape_class(shape_type).__name__)
+ if not (deleted[0] == 0 and len(client_ids_to_delete) == 0) and (class_name in deleted[1] and deleted[1][class_name] != len(client_ids_to_delete)):
+ raise Exception('Number of deleted object doesn\'t match with requested number')
+
+ def _delete_paths_from_db(self):
+ for shape_type in ['polygon_paths', 'polyline_paths', 'points_paths', 'box_paths']:
+ client_ids_to_delete = list(shape.client_id for shape in getattr(self, shape_type))
+ deleted = self.db_job.objectpath_set.filter(client_id__in=client_ids_to_delete).delete()
+ class_name = 'engine.ObjectPath'
+ if not (deleted[0] == 0 and len(client_ids_to_delete) == 0) and \
+ (class_name in deleted[1] and deleted[1][class_name] != len(client_ids_to_delete)):
+ raise Exception('Number of deleted object doesn\'t match with requested number')
+
+ def _delete_all_shapes_from_db(self):
+ for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
+ self._get_shape_set(shape_type).all().delete()
+
+ def _delete_all_paths_from_db(self):
+ self.db_job.objectpath_set.all().delete()
+
+ def save_to_db(self, action):
+ if action == 'create':
+ self._save_shapes_to_db()
+ self._save_paths_to_db()
+ elif action == 'update':
+ self._update_shapes_in_db()
+ self._update_paths_in_db()
+ elif action == 'delete':
+ self._delete_shapes_from_db()
+ self._delete_paths_from_db()
+
+ def delete_objs_from_db(self):
+ self._delete_all_shapes_from_db()
+ self._delete_all_paths_from_db()
+
def to_client(self):
data = {
"boxes": [],
@@ -1097,6 +1319,7 @@ def to_client(self):
for box in self.boxes:
data["boxes"].append({
+ "client_id": box.client_id,
"label_id": box.label.id,
"group_id": box.group_id,
"xtl": box.xtl,
@@ -1112,6 +1335,7 @@ def to_client(self):
for poly_type in ['polygons', 'polylines', 'points']:
for poly in getattr(self, poly_type):
data[poly_type].append({
+ "client_id": poly.client_id,
"label_id": poly.label.id,
"group_id": poly.group_id,
"points": poly.points,
@@ -1123,6 +1347,7 @@ def to_client(self):
for box_path in self.box_paths:
data["box_paths"].append({
+ "client_id": box_path.client_id,
"label_id": box_path.label.id,
"group_id": box_path.group_id,
"frame": box_path.frame,
@@ -1145,6 +1370,7 @@ def to_client(self):
for poly_path_type in ['polygon_paths', 'polyline_paths', 'points_paths']:
for poly_path in getattr(self, poly_path_type):
data[poly_path_type].append({
+ "client_id": poly_path.client_id,
"label_id": poly_path.label.id,
"group_id": poly_path.group_id,
"frame": poly_path.frame,
@@ -1163,6 +1389,25 @@ def to_client(self):
return data
+ def validate_data_from_client(self, data):
+ # check unique id for each object
+ client_ids = set()
+ def extract_and_check_clinet_id(shape):
+ if 'client_id' not in shape:
+ raise Exception('No client_id field in received data')
+ client_id = shape['client_id']
+ if client_id in client_ids:
+ raise Exception('More than one object has the same client_id {}'.format(client_id))
+ client_ids.add(client_id)
+ return client_id
+
+ shape_types = ['boxes', 'points', 'polygons', 'polylines', 'box_paths',
+ 'points_paths', 'polygon_paths', 'polyline_paths']
+
+ for action in ['create', 'update', 'delete']:
+ for shape_type in shape_types:
+ for shape in data[action][shape_type]:
+ extract_and_check_clinet_id(shape)
class _AnnotationForSegment(_Annotation):
def __init__(self, db_segment):
@@ -1651,23 +1896,26 @@ def _flip_shape(shape, im_w, im_h):
shapes["polygons"] = {}
shapes["polylines"] = {}
shapes["points"] = {}
-
- for box in self.to_boxes():
+ boxes, max_client_id = self.to_boxes(self.get_max_client_id() + 1)
+ for box in boxes:
if box.frame not in shapes["boxes"]:
shapes["boxes"][box.frame] = []
shapes["boxes"][box.frame].append(box)
- for polygon in self.to_polygons():
+ polygons, max_client_id = self.to_polygons(max_client_id)
+ for polygon in polygons:
if polygon.frame not in shapes["polygons"]:
shapes["polygons"][polygon.frame] = []
shapes["polygons"][polygon.frame].append(polygon)
- for polyline in self.to_polylines():
+ polylines, max_client_id = self.to_polylines(max_client_id)
+ for polyline in polylines:
if polyline.frame not in shapes["polylines"]:
shapes["polylines"][polyline.frame] = []
shapes["polylines"][polyline.frame].append(polyline)
- for points in self.to_points():
+ points, max_client_id = self.to_points(max_client_id)
+ for points in points:
if points.frame not in shapes["points"]:
shapes["points"][points.frame] = []
shapes["points"][points.frame].append(points)
@@ -1707,7 +1955,8 @@ def _flip_shape(shape, im_w, im_h):
("ytl", "{:.2f}".format(shape.ytl)),
("xbr", "{:.2f}".format(shape.xbr)),
("ybr", "{:.2f}".format(shape.ybr)),
- ("occluded", str(int(shape.occluded)))
+ ("occluded", str(int(shape.occluded))),
+ ("client_id", str(shape.client_id)),
])
if db_task.z_order:
dump_dict['z_order'] = str(shape.z_order)
@@ -1726,7 +1975,8 @@ def _flip_shape(shape, im_w, im_h):
"{:.2f}".format(float(p.split(',')[1]))
)) for p in shape.points.split(' '))
)),
- ("occluded", str(int(shape.occluded)))
+ ("occluded", str(int(shape.occluded))),
+ ("client_id", str(shape.client_id)),
])
if db_task.z_order:
@@ -1773,7 +2023,8 @@ def _flip_shape(shape, im_w, im_h):
for path in path_list:
dump_dict = OrderedDict([
("id", str(path_idx)),
- ("label", path.label.name)
+ ("label", path.label.name),
+ ("client_id", str(path.client_id)),
])
if path.group_id:
dump_dict['group_id'] = str(path.group_id)
diff --git a/cvat/apps/engine/migrations/0010_auto_20181011_1517.py b/cvat/apps/engine/migrations/0010_auto_20181011_1517.py
new file mode 100644
index 000000000000..c33d1319c0f4
--- /dev/null
+++ b/cvat/apps/engine/migrations/0010_auto_20181011_1517.py
@@ -0,0 +1,38 @@
+# Generated by Django 2.0.9 on 2018-10-11 12:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('engine', '0009_auto_20180917_1424'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='labeledbox',
+ name='client_id',
+ field=models.BigIntegerField(default=-1),
+ ),
+ migrations.AddField(
+ model_name='labeledpoints',
+ name='client_id',
+ field=models.BigIntegerField(default=-1),
+ ),
+ migrations.AddField(
+ model_name='labeledpolygon',
+ name='client_id',
+ field=models.BigIntegerField(default=-1),
+ ),
+ migrations.AddField(
+ model_name='labeledpolyline',
+ name='client_id',
+ field=models.BigIntegerField(default=-1),
+ ),
+ migrations.AddField(
+ model_name='objectpath',
+ name='client_id',
+ field=models.BigIntegerField(default=-1),
+ ),
+ ]
diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py
index 6883018b194b..608cdafb1e30 100644
--- a/cvat/apps/engine/models.py
+++ b/cvat/apps/engine/models.py
@@ -139,6 +139,7 @@ class Annotation(models.Model):
label = models.ForeignKey(Label, on_delete=models.CASCADE)
frame = models.PositiveIntegerField()
group_id = models.PositiveIntegerField(default=0)
+ client_id = models.BigIntegerField(default=-1)
class Meta:
abstract = True
diff --git a/cvat/apps/engine/static/engine/js/3rdparty/md5.js b/cvat/apps/engine/static/engine/js/3rdparty/md5.js
deleted file mode 100644
index 762de3c52012..000000000000
--- a/cvat/apps/engine/static/engine/js/3rdparty/md5.js
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * JavaScript MD5
- * https://github.com/blueimp/JavaScript-MD5
- *
- * Copyright 2011, Sebastian Tschan
- * https://blueimp.net
- *
- * Licensed under the MIT license:
- * https://opensource.org/licenses/MIT
- *
- * Based on
- * A JavaScript implementation of the RSA Data Security, Inc. MD5 Message
- * Digest Algorithm, as defined in RFC 1321.
- * Version 2.2 Copyright (C) Paul Johnston 1999 - 2009
- * Other contributors: Greg Holt, Andrew Kepert, Ydnar, Lostinet
- * Distributed under the BSD License
- * See http://pajhome.org.uk/crypt/md5 for more info.
- */
-
-/* global define */
-
-;(function ($) {
- 'use strict'
-
- /*
- * Add integers, wrapping at 2^32. This uses 16-bit operations internally
- * to work around bugs in some JS interpreters.
- */
- function safeAdd (x, y) {
- var lsw = (x & 0xffff) + (y & 0xffff)
- var msw = (x >> 16) + (y >> 16) + (lsw >> 16)
- return (msw << 16) | (lsw & 0xffff)
- }
-
- /*
- * Bitwise rotate a 32-bit number to the left.
- */
- function bitRotateLeft (num, cnt) {
- return (num << cnt) | (num >>> (32 - cnt))
- }
-
- /*
- * These functions implement the four basic operations the algorithm uses.
- */
- function md5cmn (q, a, b, x, s, t) {
- return safeAdd(bitRotateLeft(safeAdd(safeAdd(a, q), safeAdd(x, t)), s), b)
- }
- function md5ff (a, b, c, d, x, s, t) {
- return md5cmn((b & c) | (~b & d), a, b, x, s, t)
- }
- function md5gg (a, b, c, d, x, s, t) {
- return md5cmn((b & d) | (c & ~d), a, b, x, s, t)
- }
- function md5hh (a, b, c, d, x, s, t) {
- return md5cmn(b ^ c ^ d, a, b, x, s, t)
- }
- function md5ii (a, b, c, d, x, s, t) {
- return md5cmn(c ^ (b | ~d), a, b, x, s, t)
- }
-
- /*
- * Calculate the MD5 of an array of little-endian words, and a bit length.
- */
- function binlMD5 (x, len) {
- /* append padding */
- x[len >> 5] |= 0x80 << (len % 32)
- x[((len + 64) >>> 9 << 4) + 14] = len
-
- var i
- var olda
- var oldb
- var oldc
- var oldd
- var a = 1732584193
- var b = -271733879
- var c = -1732584194
- var d = 271733878
-
- for (i = 0; i < x.length; i += 16) {
- olda = a
- oldb = b
- oldc = c
- oldd = d
-
- a = md5ff(a, b, c, d, x[i], 7, -680876936)
- d = md5ff(d, a, b, c, x[i + 1], 12, -389564586)
- c = md5ff(c, d, a, b, x[i + 2], 17, 606105819)
- b = md5ff(b, c, d, a, x[i + 3], 22, -1044525330)
- a = md5ff(a, b, c, d, x[i + 4], 7, -176418897)
- d = md5ff(d, a, b, c, x[i + 5], 12, 1200080426)
- c = md5ff(c, d, a, b, x[i + 6], 17, -1473231341)
- b = md5ff(b, c, d, a, x[i + 7], 22, -45705983)
- a = md5ff(a, b, c, d, x[i + 8], 7, 1770035416)
- d = md5ff(d, a, b, c, x[i + 9], 12, -1958414417)
- c = md5ff(c, d, a, b, x[i + 10], 17, -42063)
- b = md5ff(b, c, d, a, x[i + 11], 22, -1990404162)
- a = md5ff(a, b, c, d, x[i + 12], 7, 1804603682)
- d = md5ff(d, a, b, c, x[i + 13], 12, -40341101)
- c = md5ff(c, d, a, b, x[i + 14], 17, -1502002290)
- b = md5ff(b, c, d, a, x[i + 15], 22, 1236535329)
-
- a = md5gg(a, b, c, d, x[i + 1], 5, -165796510)
- d = md5gg(d, a, b, c, x[i + 6], 9, -1069501632)
- c = md5gg(c, d, a, b, x[i + 11], 14, 643717713)
- b = md5gg(b, c, d, a, x[i], 20, -373897302)
- a = md5gg(a, b, c, d, x[i + 5], 5, -701558691)
- d = md5gg(d, a, b, c, x[i + 10], 9, 38016083)
- c = md5gg(c, d, a, b, x[i + 15], 14, -660478335)
- b = md5gg(b, c, d, a, x[i + 4], 20, -405537848)
- a = md5gg(a, b, c, d, x[i + 9], 5, 568446438)
- d = md5gg(d, a, b, c, x[i + 14], 9, -1019803690)
- c = md5gg(c, d, a, b, x[i + 3], 14, -187363961)
- b = md5gg(b, c, d, a, x[i + 8], 20, 1163531501)
- a = md5gg(a, b, c, d, x[i + 13], 5, -1444681467)
- d = md5gg(d, a, b, c, x[i + 2], 9, -51403784)
- c = md5gg(c, d, a, b, x[i + 7], 14, 1735328473)
- b = md5gg(b, c, d, a, x[i + 12], 20, -1926607734)
-
- a = md5hh(a, b, c, d, x[i + 5], 4, -378558)
- d = md5hh(d, a, b, c, x[i + 8], 11, -2022574463)
- c = md5hh(c, d, a, b, x[i + 11], 16, 1839030562)
- b = md5hh(b, c, d, a, x[i + 14], 23, -35309556)
- a = md5hh(a, b, c, d, x[i + 1], 4, -1530992060)
- d = md5hh(d, a, b, c, x[i + 4], 11, 1272893353)
- c = md5hh(c, d, a, b, x[i + 7], 16, -155497632)
- b = md5hh(b, c, d, a, x[i + 10], 23, -1094730640)
- a = md5hh(a, b, c, d, x[i + 13], 4, 681279174)
- d = md5hh(d, a, b, c, x[i], 11, -358537222)
- c = md5hh(c, d, a, b, x[i + 3], 16, -722521979)
- b = md5hh(b, c, d, a, x[i + 6], 23, 76029189)
- a = md5hh(a, b, c, d, x[i + 9], 4, -640364487)
- d = md5hh(d, a, b, c, x[i + 12], 11, -421815835)
- c = md5hh(c, d, a, b, x[i + 15], 16, 530742520)
- b = md5hh(b, c, d, a, x[i + 2], 23, -995338651)
-
- a = md5ii(a, b, c, d, x[i], 6, -198630844)
- d = md5ii(d, a, b, c, x[i + 7], 10, 1126891415)
- c = md5ii(c, d, a, b, x[i + 14], 15, -1416354905)
- b = md5ii(b, c, d, a, x[i + 5], 21, -57434055)
- a = md5ii(a, b, c, d, x[i + 12], 6, 1700485571)
- d = md5ii(d, a, b, c, x[i + 3], 10, -1894986606)
- c = md5ii(c, d, a, b, x[i + 10], 15, -1051523)
- b = md5ii(b, c, d, a, x[i + 1], 21, -2054922799)
- a = md5ii(a, b, c, d, x[i + 8], 6, 1873313359)
- d = md5ii(d, a, b, c, x[i + 15], 10, -30611744)
- c = md5ii(c, d, a, b, x[i + 6], 15, -1560198380)
- b = md5ii(b, c, d, a, x[i + 13], 21, 1309151649)
- a = md5ii(a, b, c, d, x[i + 4], 6, -145523070)
- d = md5ii(d, a, b, c, x[i + 11], 10, -1120210379)
- c = md5ii(c, d, a, b, x[i + 2], 15, 718787259)
- b = md5ii(b, c, d, a, x[i + 9], 21, -343485551)
-
- a = safeAdd(a, olda)
- b = safeAdd(b, oldb)
- c = safeAdd(c, oldc)
- d = safeAdd(d, oldd)
- }
- return [a, b, c, d]
- }
-
- /*
- * Convert an array of little-endian words to a string
- */
- function binl2rstr (input) {
- var i
- var output = ''
- var length32 = input.length * 32
- for (i = 0; i < length32; i += 8) {
- output += String.fromCharCode((input[i >> 5] >>> (i % 32)) & 0xff)
- }
- return output
- }
-
- /*
- * Convert a raw string to an array of little-endian words
- * Characters >255 have their high-byte silently ignored.
- */
- function rstr2binl (input) {
- var i
- var output = []
- output[(input.length >> 2) - 1] = undefined
- for (i = 0; i < output.length; i += 1) {
- output[i] = 0
- }
- var length8 = input.length * 8
- for (i = 0; i < length8; i += 8) {
- output[i >> 5] |= (input.charCodeAt(i / 8) & 0xff) << (i % 32)
- }
- return output
- }
-
- /*
- * Calculate the MD5 of a raw string
- */
- function rstrMD5 (s) {
- return binl2rstr(binlMD5(rstr2binl(s), s.length * 8))
- }
-
- /*
- * Calculate the HMAC-MD5, of a key and some data (raw strings)
- */
- function rstrHMACMD5 (key, data) {
- var i
- var bkey = rstr2binl(key)
- var ipad = []
- var opad = []
- var hash
- ipad[15] = opad[15] = undefined
- if (bkey.length > 16) {
- bkey = binlMD5(bkey, key.length * 8)
- }
- for (i = 0; i < 16; i += 1) {
- ipad[i] = bkey[i] ^ 0x36363636
- opad[i] = bkey[i] ^ 0x5c5c5c5c
- }
- hash = binlMD5(ipad.concat(rstr2binl(data)), 512 + data.length * 8)
- return binl2rstr(binlMD5(opad.concat(hash), 512 + 128))
- }
-
- /*
- * Convert a raw string to a hex string
- */
- function rstr2hex (input) {
- var hexTab = '0123456789abcdef'
- var output = ''
- var x
- var i
- for (i = 0; i < input.length; i += 1) {
- x = input.charCodeAt(i)
- output += hexTab.charAt((x >>> 4) & 0x0f) + hexTab.charAt(x & 0x0f)
- }
- return output
- }
-
- /*
- * Encode a string as utf-8
- */
- function str2rstrUTF8 (input) {
- return unescape(encodeURIComponent(input))
- }
-
- /*
- * Take string arguments and return either raw or hex encoded strings
- */
- function rawMD5 (s) {
- return rstrMD5(str2rstrUTF8(s))
- }
- function hexMD5 (s) {
- return rstr2hex(rawMD5(s))
- }
- function rawHMACMD5 (k, d) {
- return rstrHMACMD5(str2rstrUTF8(k), str2rstrUTF8(d))
- }
- function hexHMACMD5 (k, d) {
- return rstr2hex(rawHMACMD5(k, d))
- }
-
- function md5 (string, key, raw) {
- if (!key) {
- if (!raw) {
- return hexMD5(string)
- }
- return rawMD5(string)
- }
- if (!raw) {
- return hexHMACMD5(key, string)
- }
- return rawHMACMD5(key, string)
- }
-
- if (typeof define === 'function' && define.amd) {
- define(function () {
- return md5
- })
- } else if (typeof module === 'object' && module.exports) {
- module.exports = md5
- } else {
- $.md5 = md5
- }
-})(this)
diff --git a/cvat/apps/engine/static/engine/js/annotationParser.js b/cvat/apps/engine/static/engine/js/annotationParser.js
index 92536516ec36..3af8511c685f 100644
--- a/cvat/apps/engine/static/engine/js/annotationParser.js
+++ b/cvat/apps/engine/static/engine/js/annotationParser.js
@@ -15,6 +15,7 @@ class AnnotationParser {
this._flipped = job.flipped;
this._im_meta = job.image_meta_data;
this._labelsInfo = labelsInfo;
+ this._client_id_set = new Set();
}
_xmlParseError(parsedXML) {
@@ -131,7 +132,8 @@ class AnnotationParser {
let result = [];
for (let track of tracks) {
let label = track.getAttribute('label');
- let group_id = track.getAttribute('group_id') || "0";
+ let group_id = track.getAttribute('group_id') || '0';
+ let client_id = track.getAttribute('client_id') || '-1';
let labelId = this._labelsInfo.labelIdOf(label);
if (labelId === null) {
throw Error(`An unknown label found in the annotation file: ${label}`);
@@ -149,6 +151,7 @@ class AnnotationParser {
!+shapes[0].getAttribute('outside') && +shapes[1].getAttribute('outside')) {
shapes[0].setAttribute('label', label);
shapes[0].setAttribute('group_id', group_id);
+ shapes[0].setAttribute('client_id', client_id);
result.push(shapes[0]);
}
}
@@ -157,6 +160,17 @@ class AnnotationParser {
return result;
}
+ _updateClientIds(data) {
+ let maxId = Math.max(-1, ...Array.from(this._client_id_set));
+ for (const shape_type in data) {
+ for (const shape of data[shape_type]) {
+ if (shape.client_id === -1) {
+ shape.client_id = ++maxId;
+ }
+ }
+ }
+ }
+
_parseAnnotationData(xml) {
let data = {
boxes: [],
@@ -209,6 +223,15 @@ class AnnotationParser {
throw Error('An unknown label found in the annotation file: ' + shape.getAttribute('label'));
}
+ let client_id = parseInt(shape.getAttribute('client_id') || '-1');
+ if (client_id !== -1) {
+ if (this._client_id_set.has(client_id)) {
+ throw Error('More than one shape has the same client_id attribute');
+ }
+
+ this._client_id_set.add(client_id);
+ }
+
let attributeList = this._getAttributeList(shape, labelId);
if (shape_type === 'boxes') {
@@ -224,6 +247,7 @@ class AnnotationParser {
ybr: ybr,
z_order: z_order,
attributes: attributeList,
+ client_id: client_id,
});
}
else {
@@ -236,6 +260,7 @@ class AnnotationParser {
occluded: occluded,
z_order: z_order,
attributes: attributeList,
+ client_id: client_id,
});
}
}
@@ -255,7 +280,8 @@ class AnnotationParser {
let tracks = xml.getElementsByTagName('track');
for (let track of tracks) {
let labelId = this._labelsInfo.labelIdOf(track.getAttribute('label'));
- let groupId = track.getAttribute('group_id') || "0";
+ let groupId = track.getAttribute('group_id') || '0';
+ let client_id = parseInt(track.getAttribute('client_id') || '-1');
if (labelId === null) {
throw Error('An unknown label found in the annotation file: ' + name);
}
@@ -307,9 +333,18 @@ class AnnotationParser {
group_id: +groupId,
frame: +parsed[type][0].getAttribute('frame'),
attributes: [],
- shapes: []
+ shapes: [],
+ client_id: client_id,
};
+ if (client_id !== -1) {
+ if (this._client_id_set.has(client_id)) {
+ throw Error('More than one shape has the same client_id attribute');
+ }
+
+ this._client_id_set.add(client_id);
+ }
+
for (let shape of parsed[type]) {
let keyFrame = +shape.getAttribute('keyframe');
let outside = +shape.getAttribute('outside');
@@ -381,7 +416,12 @@ class AnnotationParser {
return data;
}
+ _reset() {
+ this._client_id_set.clear();
+ }
+
parse(text) {
+ this._reset();
let xml = this._parser.parseFromString(text, 'text/xml');
let parseerror = this._xmlParseError(xml);
if (parseerror.length) {
@@ -390,6 +430,8 @@ class AnnotationParser {
let interpolationData = this._parseInterpolationData(xml);
let annotationData = this._parseAnnotationData(xml);
- return Object.assign({}, annotationData, interpolationData);
+ let data = Object.assign({}, annotationData, interpolationData);
+ this._updateClientIds(data);
+ return data;
}
}
diff --git a/cvat/apps/engine/static/engine/js/annotationUI.js b/cvat/apps/engine/static/engine/js/annotationUI.js
index 036845e921e6..9f0ca3acb5e1 100644
--- a/cvat/apps/engine/static/engine/js/annotationUI.js
+++ b/cvat/apps/engine/static/engine/js/annotationUI.js
@@ -105,7 +105,7 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) {
let shapeCollectionView = new ShapeCollectionView(shapeCollectionModel, shapeCollectionController);
window.cvat.data = {
- get: () => shapeCollectionModel.export(),
+ get: () => shapeCollectionModel.exportAll(),
set: (data) => {
shapeCollectionModel.empty();
shapeCollectionModel.import(data);
@@ -689,11 +689,11 @@ function saveAnnotation(shapeCollectionModel, job) {
'points count': totalStat.points.annotation + totalStat.points.interpolation,
});
- let exportedData = shapeCollectionModel.export();
- let annotationLogs = Logger.getLogs();
+ const exportedData = shapeCollectionModel.export();
+ const annotationLogs = Logger.getLogs();
const data = {
- annotation: exportedData,
+ annotation: JSON.stringify(exportedData),
logs: JSON.stringify(annotationLogs.export()),
};
@@ -702,6 +702,7 @@ function saveAnnotation(shapeCollectionModel, job) {
saveJobRequest(job.jobid, data, () => {
// success
+ shapeCollectionModel.reset_state();
shapeCollectionModel.updateHash();
saveButton.text('Success!');
setTimeout(() => {
diff --git a/cvat/apps/engine/static/engine/js/base.js b/cvat/apps/engine/static/engine/js/base.js
index bfc937c4e5a6..a4e442f58ed6 100644
--- a/cvat/apps/engine/static/engine/js/base.js
+++ b/cvat/apps/engine/static/engine/js/base.js
@@ -4,7 +4,10 @@
* SPDX-License-Identifier: MIT
*/
-/* exported confirm showMessage showOverlay dumpAnnotationRequest */
+/* exported confirm showMessage showOverlay dumpAnnotationRequest ExportType
+ createExportContainer getExportTargetContainer
+*/
+
"use strict";
Math.clamp = function(x, min, max) {
@@ -160,6 +163,81 @@ function dumpAnnotationRequest(dumpButton, taskID) {
}
}
+const ExportType = Object.freeze({
+ 'create': 0,
+ 'update': 1,
+ 'delete': 2,
+});
+
+function createExportContainer() {
+ const container = {};
+ Object.keys(ExportType).forEach( action => {
+ container[action] = {
+ "boxes": [],
+ "box_paths": [],
+ "points": [],
+ "points_paths": [],
+ "polygons": [],
+ "polygon_paths": [],
+ "polylines": [],
+ "polyline_paths": [],
+ };
+ });
+ container.pre_erase = false;
+
+ return container;
+}
+
+function getExportTargetContainer(export_type, shape_type, container) {
+ let shape_container_target = undefined;
+ let export_action_container = undefined;
+
+ switch (export_type) {
+ case ExportType.create:
+ export_action_container = container.create;
+ break;
+ case ExportType.update:
+ export_action_container = container.update;
+ break;
+ case ExportType.delete:
+ export_action_container = container.delete;
+ break;
+ default:
+ throw Error('Unexpected export type');
+ }
+
+ switch (shape_type) {
+ case 'annotation_box':
+ shape_container_target = export_action_container.boxes;
+ break;
+ case 'interpolation_box':
+ shape_container_target = export_action_container.box_paths;
+ break;
+ case 'annotation_points':
+ shape_container_target = export_action_container.points;
+ break;
+ case 'interpolation_points':
+ shape_container_target = export_action_container.points_paths;
+ break;
+ case 'annotation_polygon':
+ shape_container_target = export_action_container.polygons;
+ break;
+ case 'interpolation_polygon':
+ shape_container_target = export_action_container.polygon_paths;
+ break;
+ case 'annotation_polyline':
+ shape_container_target = export_action_container.polylines;
+ break;
+ case 'interpolation_polyline':
+ shape_container_target = export_action_container.polyline_paths;
+ break;
+ default:
+ throw Error('Undefined shape type');
+ }
+
+ return shape_container_target;
+}
+
/* These HTTP methods do not require CSRF protection */
function csrfSafeMethod(method) {
return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
diff --git a/cvat/apps/engine/static/engine/js/shapeCollection.js b/cvat/apps/engine/static/engine/js/shapeCollection.js
index 1d7207adf259..7f6416f866d9 100644
--- a/cvat/apps/engine/static/engine/js/shapeCollection.js
+++ b/cvat/apps/engine/static/engine/js/shapeCollection.js
@@ -51,6 +51,8 @@ class ShapeCollectionModel extends Listener {
this._colorIdx = 0;
this._filter = new FilterModel(() => this.update());
this._splitter = new ShapeSplitter();
+ this._erased = false;
+ this._initialShapes = {};
}
_nextIdx() {
@@ -237,49 +239,46 @@ class ShapeCollectionModel extends Listener {
return this;
}
+ reset_state() {
+ this._erased = false;
+ }
export() {
- let response = {
- "boxes": [],
- "box_paths": [],
- "points": [],
- "points_paths": [],
- "polygons": [],
- "polygon_paths": [],
- "polylines": [],
- "polyline_paths": [],
- };
-
- for (let shape of this._shapes) {
- if (shape.removed) continue;
- switch (shape.type) {
- case 'annotation_box':
- response.boxes.push(shape.export());
- break;
- case 'interpolation_box':
- response.box_paths.push(shape.export());
- break;
- case 'annotation_points':
- response.points.push(shape.export());
- break;
- case 'interpolation_points':
- response.points_paths.push(shape.export());
- break;
- case 'annotation_polygon':
- response.polygons.push(shape.export());
- break;
- case 'interpolation_polygon':
- response.polygon_paths.push(shape.export());
- break;
- case 'annotation_polyline':
- response.polylines.push(shape.export());
- break;
- case 'interpolation_polyline':
- response.polyline_paths.push(shape.export());
+ const response = createExportContainer();
+ response.pre_erase = this._erased;
+
+ for (const shape of this._shapes) {
+ let target_export_container = undefined;
+ if (!shape._removed) {
+ if (!(shape.id in this._initialShapes) || this._erased) {
+ target_export_container = getExportTargetContainer(ExportType.create, shape.type, response);
+ } else if (JSON.stringify(this._initialShapes[shape.id]) !== JSON.stringify(shape.export())) {
+ target_export_container = getExportTargetContainer(ExportType.update, shape.type, response);
+ } else {
+ continue;
+ }
}
+ else if (shape.id in this._initialShapes && !this._erased) {
+ // TODO in this case need push only id
+ target_export_container = getExportTargetContainer(ExportType.delete, shape.type, response);
+ }
+ else {
+ continue;
+ }
+
+ target_export_container.push(shape.export());
}
+ return response;
+ }
- return JSON.stringify(response);
+ exportAll() {
+ const response = createExportContainer();
+ for (const shape of this._shapes) {
+ if (!shape._removed) {
+ getExportTargetContainer(ExportType.create, shape.type, response).push(shape.export());
+ }
+ }
+ return response.create;
}
find(direction) {
@@ -338,11 +337,30 @@ class ShapeCollectionModel extends Listener {
}
hasUnsavedChanges() {
- return md5(this.export()) !== this._hash;
+ const exportData = this.export();
+ for (const actionType in ExportType) {
+ for (const shapes of Object.values(exportData[actionType])) {
+ if (shapes.length) {
+ return true;
+ }
+ }
+ }
+
+ return exportData.pre_erase;
}
updateHash() {
- this._hash = md5(this.export());
+ this._initialShapes = {};
+
+ if (this._erased) {
+ return this;
+ }
+
+ for (const shape of this._shapes) {
+ if (!shape.removed) {
+ this._initialShapes[shape.id] = shape.export();
+ }
+ }
return this;
}
@@ -352,11 +370,26 @@ class ShapeCollectionModel extends Listener {
this._shapes = [];
this._idx = 0;
this._colorIdx = 0;
+ this._erased = true;
this._interpolate();
}
add(data, type) {
- let model = buildShapeModel(data, type, this._nextIdx(), this.nextColor());
+ let id = null;
+
+ if (!('client_id' in data)) {
+ id = this._nextIdx();
+ }
+ else if (data.client_id === -1 ) {
+ this._erased = true;
+ id = this._nextIdx();
+ }
+ else {
+ id = data.client_id;
+ this._idx = Math.max(this._idx, id) + 1;
+ }
+
+ let model = buildShapeModel(data, type, id, this.nextColor());
if (type.startsWith('interpolation')) {
this._interpolationShapes.push(model);
}
diff --git a/cvat/apps/engine/static/engine/js/shapeGrouper.js b/cvat/apps/engine/static/engine/js/shapeGrouper.js
index 5ad97b17268d..7a0dccbf10e2 100644
--- a/cvat/apps/engine/static/engine/js/shapeGrouper.js
+++ b/cvat/apps/engine/static/engine/js/shapeGrouper.js
@@ -262,4 +262,4 @@ class ShapeGrouperView {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/cvat/apps/engine/static/engine/js/shapes.js b/cvat/apps/engine/static/engine/js/shapes.js
index b878736abb17..667cf97bbbd5 100644
--- a/cvat/apps/engine/static/engine/js/shapes.js
+++ b/cvat/apps/engine/static/engine/js/shapes.js
@@ -14,6 +14,7 @@ const AREA_TRESHOLD = 9;
const TEXT_MARGIN = 10;
/******************************** SHAPE MODELS ********************************/
+
class ShapeModel extends Listener {
constructor(data, positions, type, id, color) {
super('onShapeUpdate', () => this );
@@ -472,6 +473,7 @@ class ShapeModel extends Listener {
this.removed = false;
}, () => {
this.removed = true;
+
}, window.cvat.player.frames.current);
// End of undo/redo code
}
@@ -490,6 +492,7 @@ class ShapeModel extends Listener {
if (value) {
this._active = false;
}
+
this._removed = value;
this.notify('remove');
}
@@ -757,6 +760,7 @@ class BoxModel extends ShapeModel {
}
return Object.assign({}, this._positions[this._frame], {
+ client_id: this._id,
attributes: immutableAttributes,
label_id: this._label,
group_id: this._groupId,
@@ -765,6 +769,7 @@ class BoxModel extends ShapeModel {
}
else {
let boxPath = {
+ client_id: this._id,
label_id: this._label,
group_id: this._groupId,
frame: this._frame,
@@ -862,7 +867,6 @@ class PolyShapeModel extends ShapeModel {
this._setupKeyFrames();
}
-
_interpolatePosition(frame) {
if (frame in this._positions) {
return Object.assign({}, this._positions[frame], {
@@ -951,6 +955,7 @@ class PolyShapeModel extends ShapeModel {
}
return Object.assign({}, this._positions[this._frame], {
+ client_id: this._id,
attributes: immutableAttributes,
label_id: this._label,
group_id: this._groupId,
@@ -959,6 +964,7 @@ class PolyShapeModel extends ShapeModel {
}
else {
let polyPath = {
+ client_id: this._id,
label_id: this._label,
group_id: this._groupId,
frame: this._frame,
@@ -1012,16 +1018,7 @@ class PolyShapeModel extends ShapeModel {
}
static convertNumberArrayToString(arrayPoints) {
- let serializedPoints = '';
- for (let point of arrayPoints) {
- serializedPoints += `${point.x},${point.y} `;
- }
- let len = serializedPoints.length;
- if (len) {
- serializedPoints = serializedPoints.substring(0, len - 1);
- }
-
- return serializedPoints;
+ return arrayPoints.map(point => `${point.x},${point.y}`).join(' ');
}
static importPositions(positions) {
@@ -3173,8 +3170,6 @@ class PointsView extends PolyShapeView {
}
}
-
-
function buildShapeModel(data, type, idx, color) {
switch (type) {
case 'interpolation_box':
diff --git a/cvat/apps/engine/static/engine/js/userConfig.js b/cvat/apps/engine/static/engine/js/userConfig.js
index 2076d09260bf..25feb1242d5a 100644
--- a/cvat/apps/engine/static/engine/js/userConfig.js
+++ b/cvat/apps/engine/static/engine/js/userConfig.js
@@ -349,4 +349,4 @@ class Config {
get settings() {
return JSON.parse(JSON.stringify(this._settings));
}
-}
\ No newline at end of file
+}
diff --git a/cvat/apps/engine/templates/engine/annotation.html b/cvat/apps/engine/templates/engine/annotation.html
index 67e6b1374fe2..fcdbc4260426 100644
--- a/cvat/apps/engine/templates/engine/annotation.html
+++ b/cvat/apps/engine/templates/engine/annotation.html
@@ -19,7 +19,6 @@
-
diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py
index dd3366eee07c..61a046f7592d 100644
--- a/cvat/apps/engine/views.py
+++ b/cvat/apps/engine/views.py
@@ -260,7 +260,6 @@ def save_annotation_for_job(request, jid):
return HttpResponse()
-
@login_required
@permission_required(perm=['engine.view_task', 'engine.change_annotation'], raise_exception=True)
def save_annotation_for_task(request, tid):
diff --git a/cvat/apps/tf_annotation/views.py b/cvat/apps/tf_annotation/views.py
index daaa9765ee48..7ff93125f148 100644
--- a/cvat/apps/tf_annotation/views.py
+++ b/cvat/apps/tf_annotation/views.py
@@ -105,20 +105,28 @@ def get_image_key(item):
def convert_to_cvat_format(data):
+ def create_anno_container():
+ return {
+ "boxes": [],
+ "polygons": [],
+ "polylines": [],
+ "points": [],
+ "box_paths": [],
+ "polygon_paths": [],
+ "polyline_paths": [],
+ "points_paths": [],
+ }
+
result = {
- "boxes": [],
- "polygons": [],
- "polylines": [],
- "points": [],
- "box_paths": [],
- "polygon_paths": [],
- "polyline_paths": [],
- "points_paths": [],
+ 'create': create_anno_container(),
+ 'update': create_anno_container(),
+ 'delete': create_anno_container(),
+ 'pre_erase': True,
}
for label in data:
boxes = data[label]
- for box in boxes:
- result['boxes'].append({
+ for i, box in enumerate(boxes):
+ result['create']['boxes'].append({
"label_id": label,
"frame": box[0],
"xtl": box[1],
@@ -128,12 +136,12 @@ def convert_to_cvat_format(data):
"z_order": 0,
"group_id": 0,
"occluded": False,
- "attributes": []
+ "attributes": [],
+ "client_id": i,
})
return result
-
def create_thread(id, labels_mapping):
try:
TRESHOLD = 0.5
diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt
index 108a8ab91e1e..fea2fb3b0160 100644
--- a/cvat/requirements/development.txt
+++ b/cvat/requirements/development.txt
@@ -11,4 +11,4 @@ six==1.11.0
wrapt==1.10.11
django-extensions==2.0.6
Werkzeug==0.14.1
-snakeviz==0.4.2
\ No newline at end of file
+snakeviz==0.4.2
diff --git a/tests/eslintrc.conf.js b/tests/eslintrc.conf.js
index 57614552f9bc..487df365d684 100644
--- a/tests/eslintrc.conf.js
+++ b/tests/eslintrc.conf.js
@@ -50,6 +50,9 @@ module.exports = {
'showOverlay': true,
'confirm': true,
'dumpAnnotationRequest': true,
+ 'createExportContainer': true,
+ 'ExportType': true,
+ 'getExportTargetContainer': true,
// from shapeCollection.js
'ShapeCollectionModel': true,
'ShapeCollectionController': true,
@@ -84,8 +87,6 @@ module.exports = {
'SELECT_POINT_STROKE_WIDTH': true,
// from mousetrap.js
'Mousetrap': true,
- // from md5.js
- 'md5': true,
// from platform.js
'platform': true,
// from player.js