From 6beb2782840a28f14bfc01fed61688ce0f4f7397 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Apr 2019 19:35:43 +0300 Subject: [PATCH] Implementation of dump is in progress --- cvat/apps/engine/annotation_v2.py | 416 +++++++++++----------- cvat/apps/engine/static/engine/js/base.js | 56 +-- cvat/apps/engine/tests/test_rest_api.py | 2 +- cvat/apps/engine/views.py | 24 +- 4 files changed, 250 insertions(+), 248 deletions(-) diff --git a/cvat/apps/engine/annotation_v2.py b/cvat/apps/engine/annotation_v2.py index 89d6cc90c72e..395c61aa482f 100644 --- a/cvat/apps/engine/annotation_v2.py +++ b/cvat/apps/engine/annotation_v2.py @@ -599,8 +599,9 @@ def init_from_db(self): self.reset() for db_job in self.db_jobs: - annotation = JobAnnotation(db_job) + annotation = JobAnnotation(db_job.id) annotation.init_from_db() + db_segment = db_job.segment self._merge_tags(annotation.data["tags"], db_segment.start_frame, self.db_task.overlap) self._merge_shapes(annotation.data["shapes"], db_segment.start_frame, @@ -608,15 +609,15 @@ def init_from_db(self): self._merge_tracks(annotation.data["tracks"], db_segment.start_frame, self.db_task.overlap) - def _merge_tags(tags, start_frame, overlap): + def _merge_tags(self, tags, start_frame, overlap): # FIXME: implement merge algorithm here self.data["tags"].extend(tags) - def _merge_shapes(shapes, start_frame, overlap): + def _merge_shapes(self, shapes, start_frame, overlap): # FIXME: implement merge algorithm here self.data["shapes"].extend(shapes) - def _merge_tracks(tracks, start_frame, overlap): + def _merge_tracks(self, tracks, start_frame, overlap): # FIXME: implement merge algorithm here self.data["tracks"].extend(tracks) @@ -660,7 +661,12 @@ def _flip_shape(shape, im_w, im_h): ("labels", [ ("label", OrderedDict([ ("name", db_label.name), - ("attributes", [("attribute", db_attr.text) + ("attributes", [OrderedDict([ + ("name", db_attr.name), + ("mutable", db_attr.mutable), + ("input_type", db_attr.input_type), + ("default_value", db_attr.default_value), + ("values", db_attr.values)]) for db_attr in db_label.attributespec_set.all()]) ])) for db_label in db_labels ]), @@ -694,207 +700,201 @@ def _flip_shape(shape, im_w, im_h): dumper.open_root() dumper.add_meta(meta) - if db_task.mode == "annotation": - shapes = {} - shapes["boxes"] = {} - shapes["polygons"] = {} - shapes["polylines"] = {} - shapes["points"] = {} - boxes = self.to_boxes() - for box in boxes: - if box.frame not in shapes["boxes"]: - shapes["boxes"][box.frame] = [] - shapes["boxes"][box.frame].append(box) - - polygons = self.to_polygons() - for polygon in polygons: - if polygon.frame not in shapes["polygons"]: - shapes["polygons"][polygon.frame] = [] - shapes["polygons"][polygon.frame].append(polygon) - - polylines = self.to_polylines() - for polyline in polylines: - if polyline.frame not in shapes["polylines"]: - shapes["polylines"][polyline.frame] = [] - shapes["polylines"][polyline.frame].append(polyline) - - points = self.to_points() - for points in points: - if points.frame not in shapes["points"]: - shapes["points"][points.frame] = [] - shapes["points"][points.frame].append(points) - - for frame in sorted(set(list(shapes["boxes"].keys()) + - list(shapes["polygons"].keys()) + - list(shapes["polylines"].keys()) + - list(shapes["points"].keys()))): - - link = db_task.get_frame_path(frame) - path = os.readlink(link) - - rpath = path.split(os.path.sep) - rpath = os.path.sep.join(rpath[rpath.index(".upload")+1:]) - - im_w = im_meta_data[frame]['width'] - im_h = im_meta_data[frame]['height'] - - dumper.open_image(OrderedDict([ - ("id", str(frame)), - ("name", rpath), - ("width", str(im_meta_data[frame]["width"])), - ("height", str(im_meta_data[frame]["height"])) - ])) - - for shape_type in ["boxes", "polygons", "polylines", "points"]: - shape_dict = shapes[shape_type] - if frame in shape_dict: - for shape in shape_dict[frame]: - if shape_type == "boxes": - if db_task.flipped: - _flip_box(shape, im_w, im_h) - - dump_dict = OrderedDict([ - ("label", shape.label.name), - ("xtl", "{:.2f}".format(shape.xtl)), - ("ytl", "{:.2f}".format(shape.ytl)), - ("xbr", "{:.2f}".format(shape.xbr)), - ("ybr", "{:.2f}".format(shape.ybr)), - ("occluded", str(int(shape.occluded))), - ]) - if db_task.z_order: - dump_dict['z_order'] = str(shape.z_order) - if shape.group_id: - dump_dict['group_id'] = str(shape.group_id) - dumper.open_box(dump_dict) - else: - if db_task.flipped: - _flip_shape(shape, im_w, im_h) - - dump_dict = OrderedDict([ - ("label", shape.label.name), - ("points", ';'.join(( - ','.join(( - "{:.2f}".format(float(p.split(',')[0])), - "{:.2f}".format(float(p.split(',')[1])) - )) for p in shape.points.split(' ')) - )), - ("occluded", str(int(shape.occluded))), - ]) - - if db_task.z_order: - dump_dict['z_order'] = str(shape.z_order) - if shape.group_id: - dump_dict['group_id'] = str(shape.group_id) - - if shape_type == "polygons": - dumper.open_polygon(dump_dict) - elif shape_type == "polylines": - dumper.open_polyline(dump_dict) - else: - dumper.open_points(dump_dict) - - for attr in shape.attributes: - dumper.add_attribute(OrderedDict([ - ("name", attr.name), - ("value", attr.value) - ])) - - if shape_type == "boxes": - dumper.close_box() - elif shape_type == "polygons": - dumper.close_polygon() - elif shape_type == "polylines": - dumper.close_polyline() - else: - dumper.close_points() - - dumper.close_image() - else: - paths = {} - paths["boxes"] = self.to_box_paths() - paths["polygons"] = self.to_polygon_paths() - paths["polylines"] = self.to_polyline_paths() - paths["points"] = self.to_points_paths() - - im_w = im_meta_data[0]['width'] - im_h = im_meta_data[0]['height'] - - counter = 0 - for shape_type in ["boxes", "polygons", "polylines", "points"]: - path_list = paths[shape_type] - for path in path_list: - path_id = path.client_id if path.client_id != -1 else counter - counter += 1 - dump_dict = OrderedDict([ - ("id", str(path_id)), - ("label", path.label.name), - ]) - if path.group_id: - dump_dict['group_id'] = str(path.group_id) - dumper.open_track(dump_dict) - if shape_type == "boxes": - for box in path.get_interpolated_boxes(): - if db_task.flipped: - _flip_box(box, im_w, im_h) - dump_dict = OrderedDict([ - ("frame", str(box.frame)), - ("xtl", "{:.2f}".format(box.xtl)), - ("ytl", "{:.2f}".format(box.ytl)), - ("xbr", "{:.2f}".format(box.xbr)), - ("ybr", "{:.2f}".format(box.ybr)), - ("outside", str(int(box.outside))), - ("occluded", str(int(box.occluded))), - ("keyframe", str(int(box.keyframe))) - ]) - - if db_task.z_order: - dump_dict["z_order"] = str(box.z_order) - - dumper.open_box(dump_dict) - for attr in path.attributes + box.attributes: - dumper.add_attribute(OrderedDict([ - ("name", attr.name), - ("value", attr.value) - ])) - dumper.close_box() - else: - for shape in path.get_interpolated_shapes(): - if db_task.flipped: - _flip_shape(shape, im_w, im_h) - dump_dict = OrderedDict([ - ("frame", str(shape.frame)), - ("points", ';'.join(( - ','.join(( - "{:.2f}".format(float(p.split(',')[0])), - "{:.2f}".format(float(p.split(',')[1])) - )) for p in shape.points.split(' ')) - )), - ("outside", str(int(shape.outside))), - ("occluded", str(int(shape.occluded))), - ("keyframe", str(int(shape.keyframe))) - ]) - - if db_task.z_order: - dump_dict["z_order"] = str(shape.z_order) - - if shape_type == "polygons": - dumper.open_polygon(dump_dict) - elif shape_type == "polylines": - dumper.open_polyline(dump_dict) - else: - dumper.open_points(dump_dict) - - for attr in path.attributes + shape.attributes: - dumper.add_attribute(OrderedDict([ - ("name", attr.name), - ("value", attr.value) - ])) - - if shape_type == "polygons": - dumper.close_polygon() - elif shape_type == "polylines": - dumper.close_polyline() - else: - dumper.close_points() - dumper.close_track() + # if db_task.mode == "annotation": + # shapes = {} + # shapes["boxes"] = {} + # shapes["polygons"] = {} + # shapes["polylines"] = {} + # shapes["points"] = {} + # for shape in self.to_shapes["shapes"]: + # if shape.type == models.ShapeType.RECTANGLE: + # if shape.frame not in shapes["boxes"]: + # shapes["boxes"][shape.frame] = [] + # shapes["boxes"][shape.frame].append(Rectangle(shape)) + # elif shape.type == models.ShapeType.POLYGON: + # if shape.frame not in shapes["polygons"]: + # shapes["polygons"][shape.frame] = [] + # shapes["polygons"][shape.frame].append(Polygon(shape)) + # elif shape.type == models.ShapeType.POLYLINE: + # if shape.frame not in shapes["polylines"]: + # shapes["polylines"][shape.frame] = [] + # shapes["polylines"][shape.frame].append(Polyline(shape)) + # elif shape.type == models.ShapeType.POINTS: + # if shape.frame not in shapes["points"]: + # shapes["points"][shape.frame] = [] + # shapes["points"][shape.frame].append(Points(shape)) + + # for frame in sorted(set(list(shapes["boxes"].keys()) + + # list(shapes["polygons"].keys()) + + # list(shapes["polylines"].keys()) + + # list(shapes["points"].keys()))): + + # link = db_task.get_frame_path(frame) + # path = os.readlink(link) + + # rpath = path.split(os.path.sep) + # rpath = os.path.sep.join(rpath[rpath.index(".upload")+1:]) + + # im_w = im_meta_data[frame]['width'] + # im_h = im_meta_data[frame]['height'] + + # dumper.open_image(OrderedDict([ + # ("id", str(frame)), + # ("name", rpath), + # ("width", str(im_meta_data[frame]["width"])), + # ("height", str(im_meta_data[frame]["height"])) + # ])) + + # for shape_type in ["boxes", "polygons", "polylines", "points"]: + # shape_dict = shapes[shape_type] + # if frame in shape_dict: + # for shape in shape_dict[frame]: + # if shape_type == "boxes": + # if db_task.flipped: + # _flip_box(shape, im_w, im_h) + + # dump_dict = OrderedDict([ + # ("label", shape.label.name), + # ("xtl", "{:.2f}".format(shape.xtl)), + # ("ytl", "{:.2f}".format(shape.ytl)), + # ("xbr", "{:.2f}".format(shape.xbr)), + # ("ybr", "{:.2f}".format(shape.ybr)), + # ("occluded", str(int(shape.occluded))), + # ]) + # if db_task.z_order: + # dump_dict['z_order'] = str(shape.z_order) + # if shape.group_id: + # dump_dict['group_id'] = str(shape.group_id) + # dumper.open_box(dump_dict) + # else: + # if db_task.flipped: + # _flip_shape(shape, im_w, im_h) + + # dump_dict = OrderedDict([ + # ("label", shape.label.name), + # ("points", ';'.join(( + # ','.join(( + # "{:.2f}".format(float(p.split(',')[0])), + # "{:.2f}".format(float(p.split(',')[1])) + # )) for p in shape.points.split(' ')) + # )), + # ("occluded", str(int(shape.occluded))), + # ]) + + # if db_task.z_order: + # dump_dict['z_order'] = str(shape.z_order) + # if shape.group_id: + # dump_dict['group_id'] = str(shape.group_id) + + # if shape_type == "polygons": + # dumper.open_polygon(dump_dict) + # elif shape_type == "polylines": + # dumper.open_polyline(dump_dict) + # else: + # dumper.open_points(dump_dict) + + # for attr in shape.attributes: + # dumper.add_attribute(OrderedDict([ + # ("name", attr.name), + # ("value", attr.value) + # ])) + + # if shape_type == "boxes": + # dumper.close_box() + # elif shape_type == "polygons": + # dumper.close_polygon() + # elif shape_type == "polylines": + # dumper.close_polyline() + # else: + # dumper.close_points() + + # dumper.close_image() + # else: + # paths = {} + # paths["boxes"] = self.to_box_paths() + # paths["polygons"] = self.to_polygon_paths() + # paths["polylines"] = self.to_polyline_paths() + # paths["points"] = self.to_points_paths() + + # im_w = im_meta_data[0]['width'] + # im_h = im_meta_data[0]['height'] + + # counter = 0 + # for shape_type in ["boxes", "polygons", "polylines", "points"]: + # path_list = paths[shape_type] + # for path in path_list: + # path_id = path.client_id if path.client_id != -1 else counter + # counter += 1 + # dump_dict = OrderedDict([ + # ("id", str(path_id)), + # ("label", path.label.name), + # ]) + # if path.group_id: + # dump_dict['group_id'] = str(path.group_id) + # dumper.open_track(dump_dict) + # if shape_type == "boxes": + # for box in path.get_interpolated_boxes(): + # if db_task.flipped: + # _flip_box(box, im_w, im_h) + # dump_dict = OrderedDict([ + # ("frame", str(box.frame)), + # ("xtl", "{:.2f}".format(box.xtl)), + # ("ytl", "{:.2f}".format(box.ytl)), + # ("xbr", "{:.2f}".format(box.xbr)), + # ("ybr", "{:.2f}".format(box.ybr)), + # ("outside", str(int(box.outside))), + # ("occluded", str(int(box.occluded))), + # ("keyframe", str(int(box.keyframe))) + # ]) + + # if db_task.z_order: + # dump_dict["z_order"] = str(box.z_order) + + # dumper.open_box(dump_dict) + # for attr in path.attributes + box.attributes: + # dumper.add_attribute(OrderedDict([ + # ("name", attr.name), + # ("value", attr.value) + # ])) + # dumper.close_box() + # else: + # for shape in path.get_interpolated_shapes(): + # if db_task.flipped: + # _flip_shape(shape, im_w, im_h) + # dump_dict = OrderedDict([ + # ("frame", str(shape.frame)), + # ("points", ';'.join(( + # ','.join(( + # "{:.2f}".format(float(p.split(',')[0])), + # "{:.2f}".format(float(p.split(',')[1])) + # )) for p in shape.points.split(' ')) + # )), + # ("outside", str(int(shape.outside))), + # ("occluded", str(int(shape.occluded))), + # ("keyframe", str(int(shape.keyframe))) + # ]) + + # if db_task.z_order: + # dump_dict["z_order"] = str(shape.z_order) + + # if shape_type == "polygons": + # dumper.open_polygon(dump_dict) + # elif shape_type == "polylines": + # dumper.open_polyline(dump_dict) + # else: + # dumper.open_points(dump_dict) + + # for attr in path.attributes + shape.attributes: + # dumper.add_attribute(OrderedDict([ + # ("name", attr.name), + # ("value", attr.value) + # ])) + + # if shape_type == "polygons": + # dumper.close_polygon() + # elif shape_type == "polylines": + # dumper.close_polyline() + # else: + # dumper.close_points() + # dumper.close_track() dumper.close_root() diff --git a/cvat/apps/engine/static/engine/js/base.js b/cvat/apps/engine/static/engine/js/base.js index 14673bd399f5..4918c45426f8 100644 --- a/cvat/apps/engine/static/engine/js/base.js +++ b/cvat/apps/engine/static/engine/js/base.js @@ -137,7 +137,7 @@ function dumpAnnotationRequest(dumpButton, taskID) { dumpButton.attr('disabled', true); $.ajax({ - url: '/dump/annotation/task/' + taskID, + url: '/api/v1/tasks/' + taskID + '/annotations/' + 'my_task_' + taskID, success: onDumpRequestSuccess, error: onDumpRequestError, }); @@ -150,7 +150,7 @@ function dumpAnnotationRequest(dumpButton, taskID) { if (requestSended) return; requestSended = true; $.ajax({ - url: '/check/annotation/task/' + taskID, + url: '/api/v1/tasks/' + taskID + '/annotations/' + 'my_task_' + taskID, success: onDumpCheckSuccess, error: onDumpCheckError, complete: () => requestSended = false, @@ -158,32 +158,32 @@ function dumpAnnotationRequest(dumpButton, taskID) { }, requestInterval); function onDumpCheckSuccess(data) { - if (data.state === 'created') { - clearInterval(checkInterval); - getDumpedFile(); - } - else if (data.state != 'started' ) { - clearInterval(checkInterval); - let message = 'Dump process completed with an error. ' + data.stderr; - dumpButton.attr('disabled', false); - showMessage(message); - throw Error(message); - } - - function getDumpedFile() { - $.ajax({ - url: '/download/annotation/task/' + taskID, - error: onGetDumpError, - success: () => window.location = '/download/annotation/task/' + taskID, - complete: () => dumpButton.attr('disabled', false) - }); - - function onGetDumpError(response) { - let message = 'Get the dump request error: ' + response.responseText; - showMessage(message); - throw Error(message); - } - } + // if (data.state === 'created') { + // clearInterval(checkInterval); + // getDumpedFile(); + // } + // else if (data.state != 'started' ) { + // clearInterval(checkInterval); + // let message = 'Dump process completed with an error. ' + data.stderr; + // dumpButton.attr('disabled', false); + // showMessage(message); + // throw Error(message); + // } + + // function getDumpedFile() { + // $.ajax({ + // url: '/download/annotation/task/' + taskID, + // error: onGetDumpError, + // success: () => window.location = '/download/annotation/task/' + taskID, + // complete: () => dumpButton.attr('disabled', false) + // }); + + // function onGetDumpError(response) { + // let message = 'Get the dump request error: ' + response.responseText; + // showMessage(message); + // throw Error(message); + // } + // } } function onDumpCheckError(response) { diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 6b147ad52d1a..358f4fee5c13 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -335,7 +335,7 @@ def setUpTestData(cls): "filename": "http://localhost/my_file.js", "line": 1, "column": 1, - "stack": None + "stack": "" } @mock.patch("cvat.apps.engine.views.clogger") diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 710d6fcd00de..0faf01536580 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -8,6 +8,7 @@ from ast import literal_eval import shutil import glob +from datetime import datetime from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.shortcuts import redirect, render, get_object_or_404 @@ -241,10 +242,12 @@ def annotations(self, request, pk): url_path='annotations/(?P[-\w]+)') def dump(self, request, pk, filename): queue = django_rq.get_queue("default") - timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + username = request.user.username + db_task = self.get_object() + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") file_ext = request.query_params.get("format", "xml") file_path = os.path.join(db_task.get_task_dirname(), - filename + "_{}.".format(timestamp) + "dump") + filename + ".{}.{}.".format(username, timestamp) + "dump") # Cleanup (remove old dump files) good_files = [rq_job.meta["file_path"] for rq_job in queue.get_jobs() @@ -253,32 +256,31 @@ def dump(self, request, pk, filename): for f in set(glob_files) - set(good_files): os.remove(f) - rq_id = "/api/v1/jobs/{}/annotations/{}".format(pk, filename) + rq_id = "{}@/api/v1/jobs/{}/annotations/{}".format(username, pk, filename) rq_job = queue.fetch_job(rq_id) if rq_job: - if rq_job.is_finished(): - # Different users can get the same dump file. - meta_id = "{}/sendfile".format(request.user.username) - if not rq_job.meta.get(meta_id): - rq_job.meta[meta_id] = True + if rq_job.is_finished: + if not rq_job.meta.get(rq_id): + rq_job.meta[rq_id] = True rq_job.save_meta() - db_task = self.get_object() return sendfile(request, rq_job.meta["file_path"], attachment=True, attachment_filename=filename + "." + file_ext) - elif rq_job.is_failed(): + elif rq_job.is_failed: rq_job.delete() return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR) else: return Response(status=status.HTTP_100_CONTINUE) else: - rq_job = queue.enqueue_call(func=annotation.dump, + rq_job = queue.enqueue_call(func=annotation_v2.dump_task_data, args=(pk, file_path, request.scheme, request.get_host(), request.query_params), job_id=rq_id) rq_job.meta["file_path"] = file_path rq_job.save_meta() + return Response(status=status.HTTP_100_CONTINUE) + @action(detail=True, methods=['GET'], serializer_class=RqStatusSerializer) def status(self, request, pk): response = self._get_rq_response(queue="default",