From f2dfcc039b047a64f68f8c1cf635cb34c819bdb7 Mon Sep 17 00:00:00 2001 From: zhiltsov-max Date: Wed, 26 Feb 2020 14:35:30 +0300 Subject: [PATCH] [Datumaro] Remote project export fixes (#1193) * Export project with trask name * Do not expose server paths * Fix tfrecord mask reading in tf>1.14 * Setuptools compatibility --- .../export_templates/plugins/cvat_rest_api_task_images.py | 4 +++- cvat/apps/dataset_manager/task.py | 4 +++- datumaro/datumaro/plugins/__init__.py | 0 datumaro/datumaro/plugins/coco_format/__init__.py | 0 datumaro/datumaro/plugins/cvat_format/__init__.py | 0 datumaro/datumaro/plugins/datumaro_format/__init__.py | 0 datumaro/datumaro/plugins/tf_detection_api_format/__init__.py | 0 .../datumaro/plugins/tf_detection_api_format/extractor.py | 3 ++- datumaro/datumaro/plugins/voc_format/__init__.py | 0 datumaro/datumaro/plugins/yolo_format/__init__.py | 0 10 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 datumaro/datumaro/plugins/__init__.py create mode 100644 datumaro/datumaro/plugins/coco_format/__init__.py create mode 100644 datumaro/datumaro/plugins/cvat_format/__init__.py create mode 100644 datumaro/datumaro/plugins/datumaro_format/__init__.py create mode 100644 datumaro/datumaro/plugins/tf_detection_api_format/__init__.py create mode 100644 datumaro/datumaro/plugins/voc_format/__init__.py create mode 100644 datumaro/datumaro/plugins/yolo_format/__init__.py diff --git a/cvat/apps/dataset_manager/export_templates/plugins/cvat_rest_api_task_images.py b/cvat/apps/dataset_manager/export_templates/plugins/cvat_rest_api_task_images.py index e073f3edf0e..2bf1b507e3b 100644 --- a/cvat/apps/dataset_manager/export_templates/plugins/cvat_rest_api_task_images.py +++ b/cvat/apps/dataset_manager/export_templates/plugins/cvat_rest_api_task_images.py @@ -103,10 +103,12 @@ def __init__(self, url): items = [] for entry in image_list: item_id = entry['id'] + item_filename = entry.get('name', str(item_id)) size = None if entry.get('height') and entry.get('width'): size = (entry['height'], entry['width']) - image = Image(data=self._make_image_loader(item_id), size=size) + image = Image(data=self._make_image_loader(item_id), + path=item_filename, size=size) item = datumaro.DatasetItem(id=item_id, image=image) items.append((item.id, item)) diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index db2b37c0a3b..4c5e941081a 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -94,7 +94,8 @@ def _load(self): CvatImagesDirExtractor) def _import_from_task(self, user): - self._project = Project.generate(self._project_dir) + self._project = Project.generate(self._project_dir, + config={'project_name': self._db_task.name}) self._project.add_source('task_%s_images' % self._db_task.id, { 'url': self._db_task.get_data_dirname(), @@ -185,6 +186,7 @@ def _remote_image_converter(self, save_dir, server_url=None): for db_image in self._db_task.image_set.all(): frame_info = { 'id': db_image.frame, + 'name': osp.basename(db_image.path), 'width': db_image.width, 'height': db_image.height, } diff --git a/datumaro/datumaro/plugins/__init__.py b/datumaro/datumaro/plugins/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/coco_format/__init__.py b/datumaro/datumaro/plugins/coco_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/cvat_format/__init__.py b/datumaro/datumaro/plugins/cvat_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/datumaro_format/__init__.py b/datumaro/datumaro/plugins/datumaro_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/__init__.py b/datumaro/datumaro/plugins/tf_detection_api_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py index c564f9e2d18..567392ddf8e 100644 --- a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py +++ b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py @@ -148,7 +148,8 @@ def _parse_tfrecord_file(cls, filepath, subset_name, images_dir): parsed_record['image/object/class/text'], default_value=b'').numpy() masks = tf.sparse.to_dense( - parsed_record['image/object/mask']).numpy() + parsed_record['image/object/mask'], + default_value=b'').numpy() for label, label_id in zip(labels, label_ids): label = label.decode('utf-8') diff --git a/datumaro/datumaro/plugins/voc_format/__init__.py b/datumaro/datumaro/plugins/voc_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datumaro/datumaro/plugins/yolo_format/__init__.py b/datumaro/datumaro/plugins/yolo_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d