Format updates (cvat-ai#389)

* Refactor image size uses in COCO

* Suppress cocoapi messages

* Extract common app executor for cli tests

datumaro/plugins/coco_format/converter.py

@@ -9,7 +9,6 @@
 import os
 import os.path as osp
 
-import numpy as np
 import pycocotools.mask as mask_utils
 
 from datumaro.components.converter import Converter
@@ -66,16 +65,10 @@ def _get_image_id(self, item):
         return self._context._get_image_id(item)
 
     def save_image_info(self, item, filename):
-        if item.has_image:
-            size = item.image.size
-            if size is not None:
-                h, w = size
-            else:
-                h = 0
-                w = 0
-        else:
-            h = 0
-            w = 0
+        h = 0
+        w = 0
+        if item.has_image and item.image.size:
+            h, w = item.image.size
 
         self._data['images'].append({
             'id': self._get_image_id(item),
@@ -268,7 +261,7 @@ def save_annotations(self, item):
         if not instances:
             return
 
-        if not item.has_image:
+        if not item.has_image or not item.image.size:
             log.warning("Item '%s': skipping writing instances "
                 "since no image info available" % item.id)
             return
@@ -297,10 +290,10 @@ def convert_instance(self, instance, item):
 
         area = 0
         if segmentation:
-            if item.has_image:
+            if item.has_image and item.image.size:
                 h, w = item.image.size
             else:
-                # NOTE: here we can guess the image size as
+                # Here we can guess the image size as
                 # it is only needed for the area computation
                 w = bbox[0] + bbox[2]
                 h = bbox[1] + bbox[3]
@@ -507,9 +500,7 @@ def save_annotations(self, item):
         if not masks:
             return
 
-        pan_format = mask_tools.merge_masks(
-            ((m.image, m.id) for m in masks),
-            start=np.zeros(item.image.size, dtype=np.uint32))
+        pan_format = mask_tools.merge_masks((m.image, m.id) for m in masks)
         save_image(osp.join(self._context._segmentation_dir, ann_filename),
             mask_tools.index2bgr(pan_format), create_dir=True)
 

datumaro/plugins/coco_format/extractor.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -18,6 +17,7 @@
 )
 from datumaro.util.image import Image, lazy_image, load_image
 from datumaro.util.mask_tools import bgr2index
+from datumaro.util.os_util import suppress_output
 
 from .format import CocoPath, CocoTask
 
@@ -59,13 +59,18 @@ def __init__(self, path, task, merge_instance_polygons=False, subset=None):
 
     @staticmethod
     def _make_subset_loader(path):
-        # COCO API has an 'unclosed file' warning
+        # TODO: remove pycocotools dependency?
+
+        # COCO API has an unclosed file problem, so we read json manually
         coco_api = COCO()
         with open(path, 'r', encoding='utf-8') as f:
             dataset = json.load(f)
 
         coco_api.dataset = dataset
-        coco_api.createIndex()
+
+        # suppress annoying messages about dataset reading from cocoapi
+        with suppress_output():
+            coco_api.createIndex()
         return coco_api
 
     def _load_categories(self, loader):

datumaro/util/os_util.py

@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: MIT
 
+from contextlib import (
+    ExitStack, contextmanager, redirect_stderr, redirect_stdout,
+)
 import importlib
 import os
 import os.path as osp
@@ -71,3 +74,17 @@ def split_path(path):
     parts.reverse()
 
     return parts
+
+@contextmanager
+def suppress_output(stdout: bool = True, stderr: bool = False):
+    with open(os.devnull, 'w') as devnull:
+        es = ExitStack()
+
+        if stdout:
+            es.enter_context(redirect_stdout(devnull))
+
+        if stderr:
+            es.enter_context(redirect_stderr(devnull))
+
+        with es:
+            yield

datumaro/util/test_utils.py

@@ -225,3 +225,7 @@ def test_save_and_load(test, source_dataset, converter, test_dir, importer,
     elif not compare:
         compare = compare_datasets
     compare(test, expected=target_dataset, actual=parsed_dataset, **kwargs)
+
+def run_datum(test, *args, expected_code=0):
+    from datumaro.cli.__main__ import main
+    test.assertEqual(expected_code, main(args), str(args))

tests/cli/test_image_zip_format.py

@@ -5,16 +5,13 @@
 
 import numpy as np
 
-from datumaro.cli.__main__ import main
 from datumaro.components.dataset import Dataset, DatasetItem
 from datumaro.util.test_utils import TestDir, compare_datasets
+from datumaro.util.test_utils import run_datum as run
 
 from ..requirements import Requirements, mark_requirement
 
 
-def run(test, *args, expected_code=0):
-    test.assertEqual(expected_code, main(args), str(args))
-
 def make_zip_archive(src_path, dst_path):
     with ZipFile(dst_path, 'w') as archive:
         for (dirpath, _, filenames) in os.walk(src_path):

tests/cli/test_kitti_raw_format.py

@@ -1,21 +1,18 @@
 from unittest import TestCase
 import os.path as osp
 
-from datumaro.cli.__main__ import main
 from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import (
     AnnotationType, Cuboid3d, DatasetItem, LabelCategories,
 )
 from datumaro.util.test_utils import TestDir, compare_datasets_3d
+from datumaro.util.test_utils import run_datum as run
 
 from ..requirements import Requirements, mark_requirement
 
 DUMMY_DATASET_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))],
     'tests', 'assets', 'kitti_dataset', 'kitti_raw')
 
-def run(test, *args, expected_code=0):
-    test.assertEqual(expected_code, main(args), str(args))
-
 class KittiRawIntegrationScenarios(TestCase):
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_convert_to_kitti_raw(self):
@@ -77,10 +74,8 @@ def test_can_convert_to_kitti_raw(self):
                 ),
             ], categories={AnnotationType.label: expected_label_cat})
 
-            run(self, 'import', '-f', 'kitti_raw',
-                '-i', DUMMY_DATASET_DIR, '-o', test_dir)
-
-            run(self, 'export', '-p', test_dir,
+            run(self, 'convert',
+                '-if', 'kitti_raw', '-i', DUMMY_DATASET_DIR,
                 '-f', 'sly_pointcloud', '-o', export_dir,
                 '--', '--save-images')
 

tests/cli/test_sly_point_cloud_format.py

@@ -1,21 +1,18 @@
 from unittest import TestCase
 import os.path as osp
 
-from datumaro.cli.__main__ import main
 from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import (
     AnnotationType, Cuboid3d, DatasetItem, LabelCategories,
 )
 from datumaro.util.test_utils import TestDir, compare_datasets_3d
+from datumaro.util.test_utils import run_datum as run
 
 from ..requirements import Requirements, mark_requirement
 
 DUMMY_DATASET_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))],
     'tests', 'assets', 'sly_pointcloud_dataset')
 
-def run(test, *args, expected_code=0):
-    test.assertEqual(expected_code, main(args), str(args))
-
 class SlyPointCloudIntegrationScenarios(TestCase):
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_convert_to_kitti_raw(self):
@@ -63,10 +60,8 @@ def test_can_convert_to_kitti_raw(self):
                 ),
             ], categories={AnnotationType.label: expected_label_cat})
 
-            run(self, 'import', '-f', 'sly_pointcloud',
-                '-i', DUMMY_DATASET_DIR, '-o', test_dir)
-
-            run(self, 'export', '-p', test_dir,
+            run(self, 'convert',
+                '-if', 'sly_pointcloud', '-i', DUMMY_DATASET_DIR,
                 '-f', 'kitti_raw', '-o', export_dir,
                 '--', '--save-images', '--allow-attrs')
 

tests/cli/test_voc_format.py

@@ -4,20 +4,17 @@
 
 import numpy as np
 
-from datumaro.cli.__main__ import main
 from datumaro.components.dataset import Dataset, DatasetItem
 from datumaro.components.extractor import Bbox, Label, Mask
 from datumaro.util.test_utils import TestDir, compare_datasets
+from datumaro.util.test_utils import run_datum as run
 import datumaro.plugins.voc_format.format as VOC
 
 from ..requirements import Requirements, mark_requirement
 
 DUMMY_DATASETS_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))],
     'tests', 'assets', 'voc_dataset')
 
-def run(test, *args, expected_code=0):
-    test.assertEqual(expected_code, main(args), str(args))
-
 class VocIntegrationScenarios(TestCase):
     def _test_can_save_and_load(self, project_path, source_path, expected_dataset,
             dataset_format, result_path='', label_map=None):

tests/cli/test_yolo_format.py

@@ -3,18 +3,15 @@
 
 import numpy as np
 
-from datumaro.cli.__main__ import main
 from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import AnnotationType, Bbox, DatasetItem
 from datumaro.util.test_utils import TestDir, compare_datasets
+from datumaro.util.test_utils import run_datum as run
 import datumaro.plugins.voc_format.format as VOC
 
 from ..requirements import Requirements, mark_requirement
 
 
-def run(test, *args, expected_code=0):
-    test.assertEqual(expected_code, main(args), str(args))
-
 class YoloIntegrationScenarios(TestCase):
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_can_save_and_load_yolo_dataset(self):

tests/test_cvat_format.py

@@ -5,11 +5,11 @@
 
 import numpy as np
 
+from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import (
     AnnotationType, Bbox, DatasetItem, Label, LabelCategories, Points, Polygon,
     PolyLine,
 )
-from datumaro.components.project import Dataset
 from datumaro.plugins.cvat_format.converter import CvatConverter
 from datumaro.plugins.cvat_format.extractor import CvatImporter
 from datumaro.util.image import Image

tests/test_dataset.py

@@ -1027,7 +1027,7 @@ def __iter__(self):
                     DatasetItem(4),
                 ])
 
-            def get(self, id, subset=None): #pylint: disable=redefined-builtin
+            def get(self, id, subset=None):
                 nonlocal get_called
                 get_called += 1
                 return DatasetItem(id, subset=subset)