[GSoC2024] Added quality reporting for Tag annotations

changelog.d/20240331_170807_vidit.agarwal.eee20_quality_report.md

@@ -0,0 +1,4 @@
+### Added
+
+- Quality Report calculation will now also include annotation of type Tag.
+  (<https://github.com/opencv/cvat/pull/7582>)

cvat/apps/quality_control/quality_reports.py

@@ -157,8 +157,12 @@ class ComparisonParameters(_Serializable):
         dm.AnnotationType.polygon,
         dm.AnnotationType.polyline,
         dm.AnnotationType.skeleton,
+        dm.AnnotationType.label,
     ]
 
+    non_groupable_ann_type = dm.AnnotationType.label
+    "Annotation type that can't be grouped"
+
     compare_attributes: bool = True
     "Enables or disables attribute checks"
 
@@ -1000,6 +1004,21 @@ def _match_ann_type(self, t, *args):
         else:
             return None
 
+    def match_labels(self, item_a, item_b):
+        def label_distance(a, b):
+            if a is None or b is None:
+                return 0
+            return 0.5 + (a.label == b.label) / 2
+
+        return self._match_segments(
+            dm.AnnotationType.label,
+            item_a,
+            item_b,
+            distance=label_distance,
+            label_matcher=lambda a, b: a.label == b.label,
+            dist_thresh=0.5,
+        )
+
     def _match_segments(
         self,
         t,
@@ -1010,6 +1029,7 @@ def _match_segments(
         label_matcher: Callable = None,
         a_objs: Optional[Sequence[dm.Annotation]] = None,
         b_objs: Optional[Sequence[dm.Annotation]] = None,
+        dist_thresh=None,
     ):
         if a_objs is None:
             a_objs = self._get_ann_type(t, item_a)
@@ -1028,7 +1048,11 @@ def _match_segments(
                 extra_args["label_matcher"] = label_matcher
 
             returned_values = _match_segments(
-                a_objs, b_objs, distance=distance, dist_thresh=self.iou_threshold, **extra_args
+                a_objs,
+                b_objs,
+                distance=distance,
+                dist_thresh=dist_thresh if dist_thresh is not None else self.iou_threshold,
+                **extra_args,
             )
 
         if self.return_distances:
@@ -1482,6 +1506,7 @@ def __init__(self, categories: dm.CategoriesInfo, *, settings: ComparisonParamet
             "outside",  # handled by other means
         }
         self.included_ann_types = settings.included_annotation_types
+        self.non_groupable_ann_type = settings.non_groupable_ann_type
         self._annotation_comparator = _DistanceComparator(
             categories,
             included_ann_types=set(self.included_ann_types)
@@ -1528,7 +1553,11 @@ def find_groups(
         self, item: dm.DatasetItem
     ) -> Tuple[Dict[int, List[dm.Annotation]], Dict[int, int]]:
         ann_groups = dm.ops.find_instances(
-            [ann for ann in item.annotations if ann.type in self.included_ann_types]
+            [
+                ann
+                for ann in item.annotations
+                if ann.type in self.included_ann_types and ann.type != self.non_groupable_ann_type
+            ]
         )
 
         groups = {}
@@ -1603,6 +1632,7 @@ def match_annotations(self, item_a: dm.DatasetItem, item_b: dm.DatasetItem):
         per_type_results = self._annotation_comparator.match_annotations(item_a, item_b)
 
         merged_results = [[], [], [], [], {}]
+        shape_merged_results = [[], [], [], [], {}]
         for shape_type in self.included_ann_types:
             shape_type_results = per_type_results.get(shape_type, None)
             if shape_type_results is None:
@@ -1611,9 +1641,14 @@ def match_annotations(self, item_a: dm.DatasetItem, item_b: dm.DatasetItem):
             for merged_field, field in zip(merged_results, shape_type_results[:-1]):
                 merged_field.extend(field)
 
+            if shape_type != dm.AnnotationType.label:
+                for merged_field, field in zip(shape_merged_results, shape_type_results[:-1]):
+                    merged_field.extend(field)
+                shape_merged_results[-1].update(per_type_results[shape_type][-1])
+
             merged_results[-1].update(per_type_results[shape_type][-1])
 
-        return merged_results
+        return {"all_ann_types": merged_results, "all_shape_ann_types": shape_merged_results}
 
     def get_distance(
         self, pairwise_distances, gt_ann: dm.Annotation, ds_ann: dm.Annotation
@@ -1687,21 +1722,30 @@ def _generate_frame_annotation_conflicts(
     ) -> List[AnnotationConflict]:
         conflicts = []
 
-        matches, mismatches, gt_unmatched, ds_unmatched, pairwise_distances = frame_results
+        matches, mismatches, gt_unmatched, ds_unmatched, _ = frame_results["all_ann_types"]
+        (
+            shape_matches,
+            shape_mismatches,
+            shape_gt_unmatched,
+            shape_ds_unmatched,
+            shape_pairwise_distances,
+        ) = frame_results["all_shape_ann_types"]
 
         def _get_similarity(gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> Optional[float]:
-            return self.comparator.get_distance(pairwise_distances, gt_ann, ds_ann)
+            return self.comparator.get_distance(shape_pairwise_distances, gt_ann, ds_ann)
 
         _matched_shapes = set(
-            id(shape) for shape_pair in itertools.chain(matches, mismatches) for shape in shape_pair
+            id(shape)
+            for shape_pair in itertools.chain(shape_matches, shape_mismatches)
+            for shape in shape_pair
         )
 
         def _find_closest_unmatched_shape(shape: dm.Annotation):
             this_shape_id = id(shape)
 
             this_shape_distances = []
 
-            for (gt_shape_id, ds_shape_id), dist in pairwise_distances.items():
+            for (gt_shape_id, ds_shape_id), dist in shape_pairwise_distances.items():
                 if gt_shape_id == this_shape_id:
                     other_shape_id = ds_shape_id
                 elif ds_shape_id == this_shape_id:
@@ -1728,21 +1772,21 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
                     )
                 )
 
-        for unmatched_ann in gt_unmatched:
+        for shape_unmatched_ann in gt_unmatched:
             conflicts.append(
                 AnnotationConflict(
                     frame_id=frame_id,
                     type=AnnotationConflictType.MISSING_ANNOTATION,
-                    annotation_ids=[self._dm_ann_to_ann_id(unmatched_ann, self._gt_dataset)],
+                    annotation_ids=[self._dm_ann_to_ann_id(shape_unmatched_ann, self._gt_dataset)],
                 )
             )
 
-        for unmatched_ann in ds_unmatched:
+        for shape_unmatched_ann in ds_unmatched:
             conflicts.append(
                 AnnotationConflict(
                     frame_id=frame_id,
                     type=AnnotationConflictType.EXTRA_ANNOTATION,
-                    annotation_ids=[self._dm_ann_to_ann_id(unmatched_ann, self._ds_dataset)],
+                    annotation_ids=[self._dm_ann_to_ann_id(shape_unmatched_ann, self._ds_dataset)],
                 )
             )
 
@@ -1759,14 +1803,14 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
             )
 
         resulting_distances = [
-            _get_similarity(gt_ann, ds_ann)
-            for gt_ann, ds_ann in itertools.chain(matches, mismatches)
+            _get_similarity(shape_gt_ann, shape_ds_ann)
+            for shape_gt_ann, shape_ds_ann in itertools.chain(shape_matches, shape_mismatches)
         ]
 
-        for unmatched_ann in itertools.chain(gt_unmatched, ds_unmatched):
-            matched_ann_id, similarity = _find_closest_unmatched_shape(unmatched_ann)
-            if matched_ann_id is not None:
-                _matched_shapes.add(matched_ann_id)
+        for shape_unmatched_ann in itertools.chain(shape_gt_unmatched, shape_ds_unmatched):
+            shape_matched_ann_id, similarity = _find_closest_unmatched_shape(shape_unmatched_ann)
+            if shape_matched_ann_id is not None:
+                _matched_shapes.add(shape_matched_ann_id)
             resulting_distances.append(similarity)
 
         resulting_distances = [
@@ -1842,10 +1886,12 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
         if self.settings.compare_groups:
             gt_groups, gt_group_map = self.comparator.find_groups(gt_item)
             ds_groups, ds_group_map = self.comparator.find_groups(ds_item)
-            matched_objects = matches + mismatches
-            ds_to_gt_groups = self.comparator.match_groups(gt_groups, ds_groups, matched_objects)
+            shape_matched_objects = shape_matches + shape_mismatches
+            ds_to_gt_groups = self.comparator.match_groups(
+                gt_groups, ds_groups, shape_matched_objects
+            )
 
-            for gt_ann, ds_ann in matched_objects:
+            for gt_ann, ds_ann in shape_matched_objects:
                 gt_group = gt_groups.get(gt_group_map[id(gt_ann)], [gt_ann])
                 ds_group = ds_groups.get(ds_group_map[id(ds_ann)], [ds_ann])
                 ds_gt_group = ds_to_gt_groups.get(ds_group_map[id(ds_ann)], None)
@@ -1868,12 +1914,17 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
                         )
                     )
 
-        valid_shapes_count = len(matches) + len(mismatches)
-        missing_shapes_count = len(gt_unmatched)
-        extra_shapes_count = len(ds_unmatched)
-        total_shapes_count = len(matches) + len(mismatches) + len(gt_unmatched) + len(ds_unmatched)
-        ds_shapes_count = len(matches) + len(mismatches) + len(ds_unmatched)
-        gt_shapes_count = len(matches) + len(mismatches) + len(gt_unmatched)
+        valid_shapes_count = len(shape_matches) + len(shape_mismatches)
+        missing_shapes_count = len(shape_gt_unmatched)
+        extra_shapes_count = len(shape_ds_unmatched)
+        total_shapes_count = (
+            len(shape_matches)
+            + len(shape_mismatches)
+            + len(shape_gt_unmatched)
+            + len(shape_ds_unmatched)
+        )
+        ds_shapes_count = len(shape_matches) + len(shape_mismatches) + len(shape_ds_unmatched)
+        gt_shapes_count = len(shape_matches) + len(shape_mismatches) + len(shape_gt_unmatched)
 
         valid_labels_count = len(matches)
         invalid_labels_count = len(mismatches)