Add e2e tests for cuboids

CHANGELOG.md

@@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   only images and image-level labels can be read/written
   (<https://github.com/openvinotoolkit/datumaro/pull/291>,
   <https://github.com/openvinotoolkit/datumaro/pull/315>).
-- Support for Supervisely Point Cloud dataset format (<https://github.com/openvinotoolkit/datumaro/pull/245>)
+- Support for Supervisely Point Cloud dataset format (<https://github.com/openvinotoolkit/datumaro/pull/245>, <https://github.com/openvinotoolkit/datumaro/pull/353>)
 - Support for KITTI Raw / Velodyne Points dataset format (<https://github.com/openvinotoolkit/datumaro/pull/245>)
 
 ### Changed

datumaro/plugins/kitti_raw_format/extractor.py

@@ -118,9 +118,11 @@ def _parse(cls, path):
 
                 # common tags
                 elif attr is not None and elem.tag == 'name':
+                    if not elem.text:
+                        raise ValueError("Attribute name can't be empty")
                     attr['name'] = elem.text
                 elif attr is not None and elem.tag == 'value':
-                    attr['value'] = elem.text
+                    attr['value'] = elem.text or ''
                 elif attr is not None and elem.tag == 'attribute':
                     if shape:
                         shape['attributes'][attr['name']] = attr['value']

datumaro/plugins/sly_pointcloud_format/converter.py

@@ -24,9 +24,6 @@
 
 
 class _SuperviselyPointCloudDumper:
-    SPECIAL_ATTRS = {'description', 'object',
-        'labelerLogin', 'createdAt', 'updatedAt', 'frame'}
-
     def __init__(self, extractor: IExtractor,
             context: 'SuperviselyPointCloudConverter'):
         self._extractor = extractor
@@ -139,7 +136,7 @@ def _write_item_annotations(self, item):
 
     def _export_item_attributes(self, item, item_ann_data, item_user_info):
         for attr_name, attr_value in item.attributes.items():
-            if attr_name in self.SPECIAL_ATTRS:
+            if attr_name in PointCloudPath.SPECIAL_ATTRS:
                 continue
 
             attr_value = self._encode_attr_value(attr_value)
@@ -182,7 +179,7 @@ def _export_item_annotations(self, item, item_ann_data, item_user_info):
             if not ann.type == AnnotationType.cuboid_3d:
                 continue
 
-            obj_id = cast(ann.attributes.get('object', ann.id), int)
+            obj_id = cast(ann.attributes.get('track_id', ann.id), int)
             if obj_id is None:
                 # should not be affected by reindex
                 # because it is used to match figures,
@@ -205,7 +202,7 @@ def _export_item_annotations(self, item, item_ann_data, item_user_info):
                 }
 
                 for attr_name, attr_value in ann.attributes.items():
-                    if attr_name in self.SPECIAL_ATTRS:
+                    if attr_name in PointCloudPath.SPECIAL_ATTRS:
                         continue
 
                     attr_value = self._encode_attr_value(attr_value)

datumaro/plugins/sly_pointcloud_format/extractor.py

@@ -43,8 +43,7 @@ def _parse(cls, rootpath):
                 encoding='utf-8') as f:
             meta = json.load(f)
 
-        label_cat = LabelCategories(
-            attributes=set(PointCloudPath.BUILTIN_ATTRS))
+        label_cat = LabelCategories()
         for label in meta.get('classes', []):
             label_cat.add(label['title'])
 
@@ -131,7 +130,7 @@ def _parse_tag(tag):
                     obj['classTitle'])[0]
 
                 attributes = {}
-                attributes['object'] = obj['id']
+                attributes['track_id'] = obj['id']
                 for tag in obj.get('tags', []):
                     attributes[tag['name']] = _parse_tag(tag)
                 for attr in _get_label_attrs(label):

datumaro/plugins/sly_pointcloud_format/format.py

@@ -14,4 +14,5 @@ class PointCloudPath:
     KEY_ID_FILE = 'key_id_map.json'
     META_FILE = 'meta.json'
 
-    BUILTIN_ATTRS = {'object'}
+    SPECIAL_ATTRS = {'description', 'track_id',
+        'labelerLogin', 'createdAt', 'updatedAt', 'frame'}

docs/formats/kitti_raw_user_manual.md

@@ -150,7 +150,7 @@ datum stats -p project
 
 ``` bash
 datum convert -if sly_pointcloud -i ../sly_pcd/ \
-    -f kitti_raw -o my_kitti/ -- --save-images --reindex --allow-attrs
+    -f kitti_raw -o my_kitti/ -- --save-images --allow-attrs
 ```
 
 ### Example 3. Create a custom dataset

docs/formats/sly_pointcloud_user_manual.md

@@ -18,7 +18,7 @@ This dataset format supports the following types of annotations:
 - `cuboid_3d`
 
 Supported annotation attributes:
-- `object` (read/write, integer),
+- `track_id` (read/write, integer), responsible for `object` field
 - `createdAt` (write, string),
 - `updatedAt` (write, string),
 - `labelerLogin` (write, string), responsible for the corresponding fields
@@ -154,11 +154,11 @@ dataset = Dataset.from_iterable([
         annotations=[
             Cuboid3d(id=206, label=0,
                 position=[320.86, 979.18, 1.04],
-                attributes={'occluded': False, 'object': 1, 'x': 1}),
+                attributes={'occluded': False, 'track_id': 1, 'x': 1}),
 
             Cuboid3d(id=207, label=1,
                 position=[318.19, 974.65, 1.29],
-                attributes={'occluded': True, 'object': 2}),
+                attributes={'occluded': True, 'track_id': 2}),
         ],
         pcd='path/to/pcd1.pcd',
         attributes={'frame': 0, 'description': 'zzz'}
@@ -168,7 +168,7 @@ dataset = Dataset.from_iterable([
         annotations=[
             Cuboid3d(id=208, label=1,
                 position=[23.04, 8.75, -0.78],
-                attributes={'occluded': False, 'object': 2})
+                attributes={'occluded': False, 'track_id': 2})
         ],
         pcd='path/to/pcd2.pcd', related_images=['image2.png'],
         attributes={'frame': 1}

tests/cli/test_kitti_raw_format.py

@@ -0,0 +1,90 @@
+from unittest import TestCase
+import os.path as osp
+
+from datumaro.cli.__main__ import main
+from datumaro.components.dataset import Dataset
+from datumaro.components.extractor import (
+    AnnotationType, Cuboid3d, DatasetItem, LabelCategories,
+)
+from datumaro.util.test_utils import TestDir, compare_datasets_3d
+
+from ..requirements import Requirements, mark_requirement
+
+DUMMY_DATASET_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))],
+    'tests', 'assets', 'kitti_dataset', 'kitti_raw')
+
+def run(test, *args, expected_code=0):
+    test.assertEqual(expected_code, main(args), str(args))
+
+class KittiRawIntegrationScenarios(TestCase):
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_can_convert_to_kitti_raw(self):
+        with TestDir() as test_dir:
+            export_dir = osp.join(test_dir, 'export_dir')
+            expected_label_cat = LabelCategories(attributes={'occluded'})
+            expected_label_cat.add('bus')
+            expected_label_cat.add('car')
+            expected_dataset = Dataset.from_iterable([
+                DatasetItem(id='0000000000',
+                    annotations=[
+                        Cuboid3d(position=[1, 2, 3],
+                            scale=[7.95, -3.62, -1.03],
+                            label=1, attributes={'occluded': False,
+                                'track_id': 1}),
+
+                        Cuboid3d(position=[1, 1, 0],
+                            scale=[8.34, 23.01, -0.76],
+                            label=0, attributes={'occluded': False,
+                                'track_id': 2})
+                    ],
+                    point_cloud=osp.join(export_dir, 'ds0', 'pointcloud',
+                        '0000000000.pcd'),
+                    related_images=[osp.join(export_dir, 'ds0',
+                        'related_images', '0000000000_pcd', '0000000000.png')
+                    ],
+                    attributes={'frame': 0, 'description': ''}
+                ),
+
+                DatasetItem(id='0000000001',
+                    annotations=[
+                        Cuboid3d(position=[0, 1, 0],
+                            scale=[8.34, 23.01, -0.76],
+                            rotation=[1, 1, 3],
+                            label=0, attributes={'occluded': True,
+                                'track_id': 2})
+                    ],
+                    point_cloud=osp.join(export_dir, 'ds0', 'pointcloud',
+                        '0000000001.pcd'),
+                    related_images=[osp.join(export_dir, 'ds0',
+                        'related_images', '0000000001_pcd', '0000000001.png')
+                    ],
+                    attributes={'frame': 1, 'description': ''}
+                ),
+
+                DatasetItem(id='0000000002',
+                    annotations=[
+                        Cuboid3d(position=[1, 2, 3],
+                            scale=[-9.41, 13.54, 0.24],
+                            label=1, attributes={'occluded': False,
+                                'track_id': 3})
+                    ],
+                    point_cloud=osp.join(export_dir, 'ds0', 'pointcloud',
+                        '0000000002.pcd'),
+                    related_images=[osp.join(export_dir, 'ds0',
+                        'related_images', '0000000002_pcd', '0000000002.png')
+                    ],
+                    attributes={'frame': 2, 'description': ''}
+                ),
+            ], categories={AnnotationType.label: expected_label_cat})
+
+            run(self, 'import', '-f', 'kitti_raw',
+                '-i', DUMMY_DATASET_DIR, '-o', test_dir)
+
+            run(self, 'export', '-p', test_dir,
+                '-f', 'sly_pointcloud', '-o', export_dir,
+                '--', '--save-images')
+
+            parsed_dataset = Dataset.import_from(export_dir,
+                format='sly_pointcloud')
+            compare_datasets_3d(self, expected_dataset, parsed_dataset,
+                require_point_cloud=True)

tests/cli/test_sly_point_cloud_format.py

@@ -0,0 +1,75 @@
+from unittest import TestCase
+import os.path as osp
+
+from datumaro.cli.__main__ import main
+from datumaro.components.dataset import Dataset
+from datumaro.components.extractor import (
+    AnnotationType, Cuboid3d, DatasetItem, LabelCategories,
+)
+from datumaro.util.test_utils import TestDir, compare_datasets_3d
+
+from ..requirements import Requirements, mark_requirement
+
+DUMMY_DATASET_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))],
+    'tests', 'assets', 'sly_pointcloud_dataset')
+
+def run(test, *args, expected_code=0):
+    test.assertEqual(expected_code, main(args), str(args))
+
+class SlyPointCloudIntegrationScenarios(TestCase):
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_can_convert_to_kitti_raw(self):
+        with TestDir() as test_dir:
+            export_dir = osp.join(test_dir, 'export_dir')
+            expected_label_cat = LabelCategories(attributes={'occluded'})
+            expected_label_cat.add('bus', attributes={'tag1', 'tag3'})
+            expected_label_cat.add('car', attributes={'tag1', 'tag3'})
+            expected_dataset = Dataset.from_iterable([
+                DatasetItem(id='frame1',
+                    annotations=[
+                        Cuboid3d(label=1,
+                            position=[0.47, 0.23, 0.79],
+                            scale=[0.01, 0.01, 0.01],
+                            attributes={'track_id': 2,
+                                'tag1': 'fd', 'tag3': '4s', 'occluded': False}),
+
+                        Cuboid3d(label=1,
+                            position=[0.36, 0.64, 0.93],
+                            scale=[0.01, 0.01, 0.01],
+                            attributes={'track_id': 3,
+                                'tag1': 'v12', 'tag3': '', 'occluded': False}),
+                    ],
+                    point_cloud=osp.join(export_dir, 'velodyne_points', 'data',
+                        'frame1.pcd'),
+                    related_images=[osp.join(export_dir, 'image_00', 'data',
+                        'frame1.png')
+                    ],
+                    attributes={'frame': 0}
+                ),
+
+                DatasetItem(id='frame2',
+                    annotations=[
+                        Cuboid3d(label=0,
+                            position=[0.59, 14.41, -0.61],
+                            attributes={'track_id': 1,
+                                'tag1': '', 'tag3': '', 'occluded': False})
+                    ],
+                    point_cloud=osp.join(export_dir, 'velodyne_points', 'data',
+                        'frame2.pcd'),
+                    related_images=[osp.join(export_dir, 'image_00', 'data',
+                        'frame2.png')
+                    ],
+                    attributes={'frame': 1}
+                ),
+            ], categories={AnnotationType.label: expected_label_cat})
+
+            run(self, 'import', '-f', 'sly_pointcloud',
+                '-i', DUMMY_DATASET_DIR, '-o', test_dir)
+
+            run(self, 'export', '-p', test_dir,
+                '-f', 'kitti_raw', '-o', export_dir,
+                '--', '--save-images', '--allow-attrs')
+
+            parsed_dataset = Dataset.import_from(export_dir, format='kitti_raw')
+            compare_datasets_3d(self, expected_dataset, parsed_dataset,
+                require_point_cloud=True)

tests/test_sly_pointcloud_format.py

@@ -38,7 +38,7 @@ def test_can_load(self):
         image2 = osp.join(DUMMY_DATASET_DIR,
             'ds0', 'related_images', 'frame2_pcd', 'img1.png')
 
-        label_cat = LabelCategories(attributes={'tag1', 'tag3', 'object'})
+        label_cat = LabelCategories(attributes={'tag1', 'tag3'})
         label_cat.add('car')
         label_cat.add('bus')
 
@@ -47,12 +47,12 @@ def test_can_load(self):
                 annotations=[
                     Cuboid3d(id=755220128, label=0,
                         position=[0.47, 0.23, 0.79], scale=[0.01, 0.01, 0.01],
-                        attributes={'object': 231825,
+                        attributes={'track_id': 231825,
                             'tag1': 'fd', 'tag3': '4s'}),
 
                     Cuboid3d(id=755337225, label=0,
                         position=[0.36, 0.64, 0.93], scale=[0.01, 0.01, 0.01],
-                        attributes={'object': 231831,
+                        attributes={'track_id': 231831,
                             'tag1': 'v12', 'tag3': ''}),
                 ],
                 point_cloud=pcd1, related_images=[image1],
@@ -64,7 +64,7 @@ def test_can_load(self):
                 annotations=[
                     Cuboid3d(id=216, label=1,
                         position=[0.59, 14.41, -0.61],
-                        attributes={'object': 36, 'tag1': '', 'tag3': ''})
+                        attributes={'track_id': 36, 'tag1': '', 'tag3': ''})
                 ],
                 point_cloud=pcd2, related_images=[image2],
                 attributes={'frame': 1, 'description': ''}
@@ -105,11 +105,11 @@ def test_can_save_and_load(self):
                 annotations=[
                     Cuboid3d(id=206, label=0,
                         position=[320.86, 979.18, 1.04],
-                        attributes={'occluded': False, 'object': 1, 'x': 1}),
+                        attributes={'occluded': False, 'track_id': 1, 'x': 1}),
 
                     Cuboid3d(id=207, label=1,
                         position=[318.19, 974.65, 1.29],
-                        attributes={'occluded': True, 'object': 2}),
+                        attributes={'occluded': True, 'track_id': 2}),
                 ],
                 point_cloud=self.pcd1,
                 attributes={'frame': 0, 'description': 'zzz'}
@@ -119,7 +119,7 @@ def test_can_save_and_load(self):
                 annotations=[
                     Cuboid3d(id=208, label=1,
                         position=[23.04, 8.75, -0.78],
-                        attributes={'occluded': False, 'object': 2})
+                        attributes={'occluded': False, 'track_id': 2})
                 ],
                 point_cloud=self.pcd2, related_images=[self.image2],
                 attributes={'frame': 1}
@@ -136,11 +136,12 @@ def test_can_save_and_load(self):
                     annotations=[
                         Cuboid3d(id=206, label=0,
                             position=[320.86, 979.18, 1.04],
-                            attributes={'occluded': False, 'object': 1, 'x': 1}),
+                            attributes={'occluded': False,
+                                'track_id': 1, 'x': 1}),
 
                         Cuboid3d(id=207, label=1,
                             position=[318.19, 974.65, 1.29],
-                            attributes={'occluded': True, 'object': 2}),
+                            attributes={'occluded': True, 'track_id': 2}),
                     ],
                     point_cloud=osp.join(test_dir,
                         'ds0', 'pointcloud', 'frame_1.pcd'),
@@ -150,7 +151,7 @@ def test_can_save_and_load(self):
                     annotations=[
                         Cuboid3d(id=208, label=1,
                             position=[23.04, 8.75, -0.78],
-                            attributes={'occluded': False, 'object': 2}),
+                            attributes={'occluded': False, 'track_id': 2}),
                     ],
                     point_cloud=osp.join(test_dir,
                         'ds0', 'pointcloud', 'frm2.pcd'),
@@ -201,7 +202,7 @@ def test_can_keep_undeclared_attributes(self):
             DatasetItem(id='frame_000000',
                 annotations=[
                     Cuboid3d(id=206, label=0, position=[320.86, 979.18, 1.04],
-                        attributes={'object': 1, 'occluded': False,
+                        attributes={'track_id': 1, 'occluded': False,
                             'a': 5, 'undeclared': 'y'}),
                 ],
                 attributes={'frame': 0}),
@@ -232,7 +233,7 @@ def test_can_drop_undeclared_attributes(self):
             DatasetItem(id='frame_000000',
                 annotations=[
                     Cuboid3d(id=206, label=0, position=[320.86, 979.18, 1.04],
-                        attributes={'object': 206, 'occluded': False, 'a': 5}),
+                        attributes={'track_id': 206, 'occluded': False, 'a': 5}),
                 ],
                 attributes={'frame': 0}),
         ], categories={AnnotationType.label: src_label_cat})