Fix OD benchmarks and disallow calculating DetailedPRCurves when using AnnotationType.RASTER

core/benchmarks/object-detection/benchmark_script.py

@@ -21,6 +21,7 @@
     Polygon,
     Prediction,
     Raster,
+    ValorDetectionManager,
     enums,
     evaluate_detection,
 )
@@ -235,11 +236,59 @@ def run_detailed_pr_curve_evaluation(groundtruths, predictions):
             enums.MetricType.mAR,
             enums.MetricType.mAPAveragedOverIOUs,
             enums.MetricType.PrecisionRecallCurve,
+            enums.MetricType.DetailedPrecisionRecallCurve,
         ],
     )
     return evaluation
 
 
+def run_base_evaluation_with_manager(groundtruths, predictions):
+    """Run a base evaluation (with no PR curves) using ValorDetectionManager."""
+    manager = ValorDetectionManager()
+    manager.add_data(groundtruths=groundtruths, predictions=predictions)
+    return manager.evaluate()
+
+
+def run_pr_curve_evaluation_with_manager(groundtruths, predictions):
+    """Run a base evaluation with PrecisionRecallCurve included using ValorDetectionManager."""
+    manager = ValorDetectionManager(
+        metrics_to_return=[
+            enums.MetricType.AP,
+            enums.MetricType.AR,
+            enums.MetricType.mAP,
+            enums.MetricType.APAveragedOverIOUs,
+            enums.MetricType.mAR,
+            enums.MetricType.mAPAveragedOverIOUs,
+            enums.MetricType.PrecisionRecallCurve,
+        ],
+    )
+
+    manager.add_data(groundtruths=groundtruths, predictions=predictions)
+
+    return manager.evaluate()
+
+
+def run_detailed_pr_curve_evaluation_with_manager(groundtruths, predictions):
+    """Run a base evaluation with PrecisionRecallCurve and DetailedPrecisionRecallCurve included using ValorDetectionManager."""
+
+    manager = ValorDetectionManager(
+        metrics_to_return=[
+            enums.MetricType.AP,
+            enums.MetricType.AR,
+            enums.MetricType.mAP,
+            enums.MetricType.APAveragedOverIOUs,
+            enums.MetricType.mAR,
+            enums.MetricType.mAPAveragedOverIOUs,
+            enums.MetricType.PrecisionRecallCurve,
+            enums.MetricType.DetailedPrecisionRecallCurve,
+        ],
+    )
+
+    manager.add_data(groundtruths=groundtruths, predictions=predictions)
+
+    return manager.evaluate()
+
+
 @dataclass
 class DataBenchmark:
     dtype: str
@@ -364,9 +413,13 @@ def run_benchmarking_analysis(
             # run evaluations
             eval_pr = None
             eval_detail = None
-            eval_base = run_base_evaluation(groundtruths, predictions)
+            eval_base = run_base_evaluation_with_manager(
+                groundtruths, predictions
+            )
             if compute_pr:
-                eval_pr = run_pr_curve_evaluation(groundtruths, predictions)
+                eval_pr = run_pr_curve_evaluation_with_manager(
+                    groundtruths, predictions
+                )
             if compute_detailed:
                 eval_detail = run_detailed_pr_curve_evaluation(
                     groundtruths, predictions
@@ -413,6 +466,7 @@ def run_benchmarking_analysis(
             (AnnotationType.BOX, AnnotationType.BOX),
         ],
         limits_to_test=[5000, 5000],
+        compute_detailed=False,
     )
 
     # run polygon benchmark
@@ -421,12 +475,13 @@ def run_benchmarking_analysis(
             (AnnotationType.POLYGON, AnnotationType.POLYGON),
         ],
         limits_to_test=[5000, 5000],
+        compute_detailed=False,
     )
-
     # run raster benchmark
     run_benchmarking_analysis(
         combinations=[
             (AnnotationType.RASTER, AnnotationType.RASTER),
         ],
-        limits_to_test=[500, 500],
+        limits_to_test=[500],
+        compute_detailed=False,
     )

core/tests/functional-tests/test_detection.py

@@ -1223,6 +1223,16 @@ def test_evaluate_detection_functional_test_with_rasters(
             pr_metrics[0]["value"][value][threshold][metric] == expected_value
         )
 
+    # test that we get a NotImplementedError if we try to calculate DetailedPRCurves with rasters
+    with pytest.raises(NotImplementedError):
+        evaluate_detection(
+            groundtruths=evaluate_detection_functional_test_groundtruths_with_rasters,
+            predictions=evaluate_detection_functional_test_predictions_with_rasters,
+            metrics_to_return=[
+                enums.MetricType.DetailedPrecisionRecallCurve,
+            ],
+        )
+
 
 def test_evaluate_mixed_annotations(
     evaluate_mixed_annotations_inputs: tuple,

core/tests/functional-tests/test_detection_manager.py

@@ -1002,6 +1002,21 @@ def test_evaluate_detection_functional_test_with_rasters_with_ValorDetectionMana
             pr_metrics[0]["value"][value][threshold][metric] == expected_value
         )
 
+    # test that we get a NotImplementedError if we try to calculate DetailedPRCurves with rasters
+    manager = managers.ValorDetectionManager(
+        metrics_to_return=[
+            enums.MetricType.DetailedPrecisionRecallCurve,
+        ],
+    )
+
+    manager.add_data(
+        groundtruths=evaluate_detection_functional_test_groundtruths_with_rasters,
+        predictions=evaluate_detection_functional_test_predictions_with_rasters,
+    )
+
+    with pytest.raises(NotImplementedError):
+        manager.evaluate()
+
 
 def test_evaluate_mixed_annotations_with_ValorDetectionManager(
     evaluate_mixed_annotations_inputs: tuple,

core/valor_core/detection.py

@@ -99,41 +99,19 @@ def _calculate_iou(
             joint_df["iou_"] = 0
 
     else:
-        iou_calculation_df = (
-            joint_df.assign(
-                intersection=lambda chain_df: chain_df.apply(
-                    lambda row: (
-                        0
-                        if row["converted_geometry_pd"] is None
-                        or row["converted_geometry_gt"] is None
-                        else np.logical_and(
-                            row["converted_geometry_pd"],
-                            row["converted_geometry_gt"],
-                        ).sum()
-                    ),
-                    axis=1,
-                )
-            )
-            .assign(
-                union_=lambda chain_df: chain_df.apply(
-                    lambda row: (
-                        0
-                        if row["converted_geometry_pd"] is None
-                        or row["converted_geometry_gt"] is None
-                        else np.sum(row["converted_geometry_gt"])
-                        + np.sum(row["converted_geometry_pd"])
-                        - row["intersection"]
-                    ),
-                    axis=1,
-                )
-            )
-            .assign(
-                iou_=lambda chain_df: chain_df["intersection"]
-                / chain_df["union_"]
+        intersection_ = joint_df.apply(
+            geometry.calculate_raster_intersection,
+            axis=1,
+        )
+        union_ = (
+            joint_df.apply(
+                geometry.calculate_raster_union,
+                axis=1,
             )
+            - intersection_
         )
 
-        joint_df = joint_df.join(iou_calculation_df["iou_"])
+        joint_df["iou_"] = intersection_ / union_
 
     return joint_df
 
@@ -643,6 +621,21 @@ def _calculate_detailed_pr_metrics(
     ) or (detailed_pr_joint_df is None):
         return []
 
+    if _check_if_series_contains_masks(
+        detailed_pr_joint_df.loc[
+            detailed_pr_joint_df["converted_geometry_pd"].notnull(),
+            "converted_geometry_pd",
+        ]
+    ) or _check_if_series_contains_masks(
+        detailed_pr_joint_df.loc[
+            detailed_pr_joint_df["converted_geometry_pd"].notnull(),
+            "converted_geometry_pd",
+        ]
+    ):
+        raise NotImplementedError(
+            "DetailedPrecisionRecallCurves are not yet implemented when dealing with rasters."
+        )
+
     # add confidence_threshold to the dataframe and sort
     detailed_pr_calc_df = pd.concat(
         [
@@ -1045,11 +1038,12 @@ def _create_detailed_joint_df(
         on=["datum_id", "label_key"],
         how="outer",
         suffixes=("_gt", "_pd"),
-    ).assign(
-        is_label_match=lambda chain_df: (
-            chain_df["label_id_pd"] == chain_df["label_id_gt"]
-        )
     )
+
+    detailed_joint_df["is_label_match"] = (
+        detailed_joint_df["label_id_pd"] == detailed_joint_df["label_id_gt"]
+    )
+
     detailed_joint_df = _calculate_iou(joint_df=detailed_joint_df)
     return detailed_joint_df
 

core/valor_core/geometry.py

@@ -1,5 +1,6 @@
 import numba
 import numpy as np
+import pandas as pd
 import shapely.affinity
 from shapely.geometry import Polygon as ShapelyPolygon
 
@@ -244,3 +245,42 @@ def is_rotated(bbox: list[tuple[float, float]]) -> bool:
         True if the bounding box is rotated, otherwise False.
     """
     return not is_axis_aligned(bbox) and not is_skewed(bbox)
+
+
+def calculate_raster_intersection(row: pd.Series) -> pd.Series:
+    """
+    Calculate the raster intersection for a given row in a pandas DataFrame. This function is intended to be used with .apply.
+
+    Parameters
+    ----------
+    row : pd.Series
+        A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt".
+
+    Returns
+    ----------
+    pd.Series
+        A Series indicating the intersection of two masks.
+    """
+    return np.logical_and(
+        row["converted_geometry_pd"], row["converted_geometry_gt"]
+    ).sum()
+
+
+def calculate_raster_union(row: pd.Series) -> pd.Series:
+    """
+    Calculate the raster union for a given row in a pandas DataFrame. This function is intended to be used with .apply.
+
+    Parameters
+    ----------
+    row : pd.Series
+        A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt".
+
+    Returns
+    ----------
+    pd.Series
+        A Series indicating the union of two masks.
+    """
+
+    return np.sum(row["converted_geometry_gt"]) + np.sum(
+        row["converted_geometry_pd"]
+    )