From f09cb74c246df9acfa43ec826d9042aa0e24c142 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 19 Mar 2024 12:19:25 +0200 Subject: [PATCH] quality_control: correct the per-class accuracy formula The current formula used to calculate `ConfusionMatrix.accuracy` is, in fact, not accuracy, but the Jaccard index. Replace it with the correct formula. Since the Jaccard index is a useful metric in its own right, calculate it too, but save it in another attribute of `ConfusionMatrix`. --- cvat/apps/quality_control/quality_reports.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 42714788a39b..888471790022 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -218,6 +218,7 @@ class ConfusionMatrix(_Serializable): precision: np.array recall: np.array accuracy: np.array + jaccard_index: np.array @property def axes(self): @@ -240,6 +241,7 @@ def from_dict(cls, d: dict): precision=np.asarray(d["precision"]), recall=np.asarray(d["recall"]), accuracy=np.asarray(d["accuracy"]), + jaccard_index=np.asarray(d["jaccard_index"]), ) @@ -1934,17 +1936,23 @@ def _generate_annotations_summary( matched_ann_counts = np.diag(confusion_matrix) ds_ann_counts = np.sum(confusion_matrix, axis=1) gt_ann_counts = np.sum(confusion_matrix, axis=0) + total_annotations_count = np.sum(confusion_matrix) - label_accuracies = _arr_div( + label_jaccard_indices = _arr_div( matched_ann_counts, ds_ann_counts + gt_ann_counts - matched_ann_counts ) label_precisions = _arr_div(matched_ann_counts, ds_ann_counts) label_recalls = _arr_div(matched_ann_counts, gt_ann_counts) + label_accuracies = ( + total_annotations_count # TP + TN + FP + FN + - (ds_ann_counts - matched_ann_counts) # - FP + - (gt_ann_counts - matched_ann_counts) # - FN + # ... = TP + TN + ) / (total_annotations_count or 1) valid_annotations_count = np.sum(matched_ann_counts) missing_annotations_count = np.sum(confusion_matrix[cls._UNMATCHED_IDX, :]) extra_annotations_count = np.sum(confusion_matrix[:, cls._UNMATCHED_IDX]) - total_annotations_count = np.sum(confusion_matrix) ds_annotations_count = np.sum(ds_ann_counts[: cls._UNMATCHED_IDX]) gt_annotations_count = np.sum(gt_ann_counts[: cls._UNMATCHED_IDX]) @@ -1961,6 +1969,7 @@ def _generate_annotations_summary( precision=label_precisions, recall=label_recalls, accuracy=label_accuracies, + jaccard_index=label_jaccard_indices, ), )