[GSoC2024] Fix import of outside track shapes in Datumaro-based formats

changelog.d/20240403_135916_yeek020407_switch_outside.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Formats with the custom `track_id` attribute should import `outside` track shapes properly (e.g. `COCO`, `COCO Keypoints`, `Datumaro`, `PASCAL VOC`)
+  (<https://github.com/opencv/cvat/pull/7669>)

cvat/apps/dataset_manager/bindings.py

@@ -15,6 +15,7 @@
 from typing import (Any, Callable, DefaultDict, Dict, Iterable, List, Literal, Mapping,
                     NamedTuple, Optional, OrderedDict, Sequence, Set, Tuple, Union)
 
+from attrs.converters import to_bool
 import datumaro as dm
 import defusedxml.ElementTree as ET
 import numpy as np
@@ -1953,7 +1954,8 @@ def import_dm_annotations(dm_dataset: dm.Dataset, instance_data: Union[ProjectDa
         'sly_pointcloud',
         'coco',
         'coco_instances',
-        'coco_person_keypoints'
+        'coco_person_keypoints',
+        'voc'
     ]
 
     label_cat = dm_dataset.categories()[dm.AnnotationType.label]
@@ -2031,9 +2033,9 @@ def reduce_fn(acc, v):
                     # because in some formats return type can be different
                     # from bool / None
                     # https://github.com/openvinotoolkit/datumaro/issues/719
-                    occluded = dm.util.cast(ann.attributes.pop('occluded', None), bool) is True
-                    keyframe = dm.util.cast(ann.attributes.get('keyframe', None), bool) is True
-                    outside = dm.util.cast(ann.attributes.pop('outside', None), bool) is True
+                    occluded = dm.util.cast(ann.attributes.pop('occluded', None), to_bool) is True
+                    keyframe = dm.util.cast(ann.attributes.get('keyframe', None), to_bool) is True
+                    outside = dm.util.cast(ann.attributes.pop('outside', None), to_bool) is True
 
                     track_id = ann.attributes.pop('track_id', None)
                     source = ann.attributes.pop('source').lower() \
@@ -2080,7 +2082,7 @@ def reduce_fn(acc, v):
                         ))
                         continue
 
-                    if keyframe or outside:
+                    if dm_dataset.format in track_formats:
                         if track_id not in tracks:
                             tracks[track_id] = {
                                 'label': label_cat.items[ann.label].name,
@@ -2107,11 +2109,8 @@ def reduce_fn(acc, v):
 
                         if ann.type == dm.AnnotationType.skeleton:
                             for element in ann.elements:
-                                element_keyframe = dm.util.cast(element.attributes.get('keyframe', None), bool, True)
                                 element_occluded = element.visibility[0] == dm.Points.Visibility.hidden
                                 element_outside = element.visibility[0] == dm.Points.Visibility.absent
-                                if not element_keyframe and not element_outside:
-                                    continue
 
                                 if element.label not in tracks[track_id]['elements']:
                                     tracks[track_id]['elements'][element.label] = instance_data.Track(
@@ -2120,6 +2119,7 @@ def reduce_fn(acc, v):
                                         source=source,
                                         shapes=[],
                                     )
+
                                 element_attributes = [
                                     instance_data.Attribute(name=n, value=str(v))
                                     for n, v in element.attributes.items()
@@ -2151,10 +2151,66 @@ def reduce_fn(acc, v):
                 raise CvatImportError("Image {}: can't import annotation "
                     "#{} ({}): {}".format(item.id, idx, ann.type.name, e)) from e
 
-    for track in tracks.values():
-        track['elements'] = list(track['elements'].values())
-        instance_data.add_track(instance_data.Track(**track))
+    def _validate_track_shapes(shapes):
+        new_shapes = []
+        prev_shape = None
+        # infer the keyframe shapes and keep only them
+        for shape in shapes:
+            prev_is_visible = prev_shape and not prev_shape.outside
+            cur_is_visible = shape and not shape.outside
+            if not cur_is_visible:
+                if prev_is_visible:
+                    shape = shape._replace(keyframe=True)
+                    new_shapes.append(shape)
+                prev_shape = shape
+                continue
+
+            has_gap = False
+            if prev_is_visible:
+                has_gap = prev_shape.frame + instance_data.frame_step < shape.frame
+
+            if not prev_is_visible or has_gap or shape.keyframe:
+                if has_gap:
+                    prev_shape = prev_shape._replace(outside=True, keyframe=True,
+                        frame=prev_shape.frame + instance_data.frame_step)
+                    new_shapes.append(prev_shape)
 
+                shape = shape._replace(keyframe=True)
+                new_shapes.append(shape)
+
+            prev_shape = shape
+
+        if prev_shape and not prev_shape.outside and (
+            prev_shape.frame + instance_data.frame_step <= stop_frame
+            # has a gap before the current instance segment end
+        ):
+            prev_shape = prev_shape._replace(outside=True, keyframe=True,
+                frame=prev_shape.frame + instance_data.frame_step)
+            new_shapes.append(prev_shape)
+
+        return new_shapes
+
+    stop_frame = int(instance_data.meta[instance_data.META_FIELD]['stop_frame'])
+    for track_id, track in tracks.items():
+        track['shapes'].sort(key=lambda t: t.frame)
+        track['shapes'] = _validate_track_shapes(track['shapes'])
+
+        if ann.type == dm.AnnotationType.skeleton:
+            new_elements = {}
+            for element_id, element in track['elements'].items():
+                element.shapes.sort(key=lambda t: t.frame)
+                new_element_shapes = _validate_track_shapes(element.shapes)
+                new_elements[element_id] = instance_data.Track(
+                                label=element.label,
+                                group=element.group,
+                                source=element.source,
+                                shapes=new_element_shapes,
+                            )
+            track['elements'] = new_elements
+
+        if track['shapes'] or track['elements']:
+            track['elements'] = list(track['elements'].values())
+            instance_data.add_track(instance_data.Track(**track))
 
 def import_labels_to_project(project_annotation, dataset: dm.Dataset):
     labels = []