Update Datumaro version (#4984)

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - `api/docs`, `api/swagger`, `api/schema` endpoints now allow unauthorized access (<https://github.com/opencv/cvat/pull/4928>)
+- Datumaro version (<https://github.com/opencv/cvat/pull/4984>)
 
 ### Deprecated
 - TDB
@@ -20,9 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - TDB
 
 ### Fixed
-- Removed a possibly duplicated encodeURI() calls in `server-proxy.ts` to prevent doubly encoding 
+- Removed a possibly duplicated encodeURI() calls in `server-proxy.ts` to prevent doubly encoding
 non-ascii paths while adding files from "Connected file share" (issue #4428)
-- Removed unnecessary volumes defined in docker-compose.serverless.yml 
+- Removed unnecessary volumes defined in docker-compose.serverless.yml
 (<https://github.com/openvinotoolkit/cvat/pull/4659>)
 
 ### Security

cvat/apps/dataset_manager/formats/cvat.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2018-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -409,7 +410,7 @@ def _load_items(self, parsed, image_items):
             di.subset = subset or DEFAULT_SUBSET_NAME
             di.annotations = item_desc.get('annotations')
             di.attributes = {'frame': int(frame_id)}
-            di.image = image if isinstance(image, Image) else di.image
+            di.media = image if isinstance(image, Image) else di.media
             image_items[(subset, osp.splitext(name)[0])] = di
         return image_items
 

cvat/apps/dataset_manager/formats/icdar.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -90,7 +91,7 @@ def _import(src_file, instance_data, load_data_callback=None):
     with TemporaryDirectory() as tmp_dir:
         zipfile.ZipFile(src_file).extractall(tmp_dir)
         dataset = Dataset.import_from(tmp_dir, 'icdar_word_recognition', env=dm_env)
-        dataset.transform(CaptionToLabel, 'icdar')
+        dataset.transform(CaptionToLabel, label='icdar')
         if load_data_callback is not None:
             load_data_callback(dataset, instance_data)
         import_dm_annotations(dataset, instance_data)
@@ -110,7 +111,7 @@ def _import(src_file, instance_data, load_data_callback=None):
         zipfile.ZipFile(src_file).extractall(tmp_dir)
 
         dataset = Dataset.import_from(tmp_dir, 'icdar_text_localization', env=dm_env)
-        dataset.transform(AddLabelToAnns, 'icdar')
+        dataset.transform(AddLabelToAnns, label='icdar')
         if load_data_callback is not None:
             load_data_callback(dataset, instance_data)
         import_dm_annotations(dataset, instance_data)
@@ -133,7 +134,7 @@ def _import(src_file, instance_data, load_data_callback=None):
     with TemporaryDirectory() as tmp_dir:
         zipfile.ZipFile(src_file).extractall(tmp_dir)
         dataset = Dataset.import_from(tmp_dir, 'icdar_text_segmentation', env=dm_env)
-        dataset.transform(AddLabelToAnns, 'icdar')
+        dataset.transform(AddLabelToAnns, label='icdar')
         dataset.transform('masks_to_polygons')
         if load_data_callback is not None:
             load_data_callback(dataset, instance_data)

cvat/apps/dataset_manager/formats/market1501.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -65,7 +66,7 @@ def _export(dst_file, instance_data, save_images=False):
     dataset = Dataset.from_extractors(GetCVATDataExtractor(
         instance_data, include_images=save_images), env=dm_env)
     with TemporaryDirectory() as temp_dir:
-        dataset.transform(LabelAttrToAttr, 'market-1501')
+        dataset.transform(LabelAttrToAttr, label='market-1501')
         dataset.export(temp_dir, 'market1501', save_images=save_images)
         make_zip_archive(temp_dir, dst_file)
 
@@ -75,7 +76,7 @@ def _import(src_file, instance_data, load_data_callback=None):
         zipfile.ZipFile(src_file).extractall(tmp_dir)
 
         dataset = Dataset.import_from(tmp_dir, 'market1501', env=dm_env)
-        dataset.transform(AttrToLabelAttr, 'market-1501')
+        dataset.transform(AttrToLabelAttr, label='market-1501')
         if load_data_callback is not None:
             load_data_callback(dataset, instance_data)
         import_dm_annotations(dataset, instance_data)

cvat/apps/dataset_manager/formats/transformations.py

@@ -9,7 +9,7 @@
 from pycocotools import mask as mask_utils
 
 from datumaro.components.extractor import ItemTransform
-import datumaro.components.annotation as datum_annotation
+import datumaro.components.annotation as dm
 
 class RotatedBoxesToPolygons(ItemTransform):
     def _rotate_point(self, p, angle, cx, cy):
@@ -20,7 +20,7 @@ def _rotate_point(self, p, angle, cx, cy):
 
     def transform_item(self, item):
         annotations = item.annotations[:]
-        anns = [p for p in annotations if p.type == datum_annotation.AnnotationType.bbox and p.attributes['rotation']]
+        anns = [p for p in annotations if p.type == dm.AnnotationType.bbox and p.attributes['rotation']]
         for ann in anns:
             rotation = math.radians(ann.attributes['rotation'])
             x0, y0, x1, y1 = ann.points
@@ -30,7 +30,7 @@ def transform_item(self, item):
             ))
 
             annotations.remove(ann)
-            annotations.append(datum_annotation.Polygon(anno_points,
+            annotations.append(dm.Polygon(anno_points,
                 label=ann.label, attributes=ann.attributes, group=ann.group,
                 z_order=ann.z_order))
 
@@ -48,5 +48,5 @@ def convert_ellipse(ellipse, img_h, img_w):
         mat = np.zeros((img_h, img_w), dtype=np.uint8)
         cv2.ellipse(mat, center, axis, angle, 0, 360, 255, thickness=-1)
         rle = mask_utils.encode(np.asfortranarray(mat))
-        return datum_annotation.RleMask(rle=rle, label=ellipse.label, z_order=ellipse.z_order,
+        return dm.RleMask(rle=rle, label=ellipse.label, z_order=ellipse.z_order,
             attributes=ellipse.attributes, group=ellipse.group)

cvat/apps/dataset_manager/formats/vggface2.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -29,7 +30,7 @@ def _import(src_file, instance_data, load_data_callback=None):
         zipfile.ZipFile(src_file).extractall(tmp_dir)
 
         dataset = Dataset.import_from(tmp_dir, 'vgg_face2', env=dm_env)
-        dataset.transform('rename', r"|([^/]+/)?(.+)|\2|")
+        dataset.transform('rename', regex=r"|([^/]+/)?(.+)|\2|")
         if load_data_callback is not None:
             load_data_callback(dataset, instance_data)
         import_dm_annotations(dataset, instance_data)

cvat/apps/dataset_manager/tests/assets/annotations.json

@@ -533,15 +533,7 @@
   },
   "Market-1501 1.0": {
     "version": 0,
-    "tags": [
-      {
-        "frame": 1,
-        "label_id": null,
-        "group": 0,
-        "source": "manual",
-        "attributes": []
-      }
-    ],
+    "tags": [],
     "shapes": [],
     "tracks": []
   },

cvat/apps/dataset_manager/tests/assets/tasks.json

@@ -336,7 +336,14 @@
         "name": "skeleton",
         "color": "#2080c0",
         "type": "skeleton",
-        "attributes": [],
+        "attributes": [
+          {
+            "name": "attr",
+            "mutable": false,
+            "input_type": "select",
+            "values": ["0", "1", "2"]
+          }
+        ],
         "sublabels": [
           {
             "name": "1",
@@ -353,14 +360,7 @@
           {
             "name": "3",
             "color": "#479ffe",
-            "attributes": [
-              {
-                "name": "attr",
-                "mutable": false,
-                "input_type": "select",
-                "values": ["0", "1", "2"]
-              }
-            ],
+            "attributes": [],
             "type": "points"
           }
         ],

cvat/apps/dataset_manager/tests/test_formats.py

@@ -1,5 +1,6 @@
 
 # Copyright (C) 2020-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -617,6 +618,14 @@ def _generate_task_images(self, count, name="image", **image_params):
         images["image_quality"] = 75
         return images
 
+    def _generate_task_images_by_names(self, names, **image_params):
+        images = {
+            f"client_files[{i}]": generate_image_file(f"{name}.jpg", **image_params)
+            for i, name in enumerate(names)
+        }
+        images["image_quality"] = 75
+        return images
+
     def _generate_task(self, images, annotation_format, **overrides):
         labels = []
         if annotation_format in ["ICDAR Recognition 1.0",
@@ -911,7 +920,10 @@ def test_can_import_annotations_for_image_with_dots_in_filename(self):
         for f in dm.views.get_import_formats():
             format_name = f.DISPLAY_NAME
 
-            images = self._generate_task_images(3, "img0.0.0")
+            if format_name == "Market-1501 1.0":
+                images = self._generate_task_images_by_names(["img0.0.0_0", "1.0_c3s1_000000_00", "img0.0.0_1"])
+            else:
+                images = self._generate_task_images(3, "img0.0.0")
             task = self._generate_task(images, format_name)
             self._generate_annotations(task, format_name)
 

cvat/requirements/base.txt

@@ -45,9 +45,7 @@ diskcache==5.0.2
 boto3==1.17.61
 azure-storage-blob==12.13.0
 google-cloud-storage==1.42.0
-# --no-binary=datumaro: workaround for pip to install
-# opencv-headless instead of regular opencv, to actually run setup script
-datumaro==0.2.0 --no-binary=datumaro
+git+https://github.com/cvat-ai/datumaro.git@bac20235bd6c792b4d068a54fd3ad14be50bb26f
 urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability
 natsort==8.0.0
 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability