From dbb190c0e3fc19a3c123bb69ea471efe1bd7e399 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 10:37:18 -0600 Subject: [PATCH 1/8] disallow rasters with detailed pr curves; fix bug --- .../object-detection/benchmark_script.py | 61 +++++++++++++++-- core/tests/functional-tests/test_detection.py | 10 +++ .../test_detection_manager.py | 15 +++++ core/valor_core/detection.py | 66 +++++++++---------- core/valor_core/geometry.py | 40 +++++++++++ 5 files changed, 152 insertions(+), 40 deletions(-) diff --git a/core/benchmarks/object-detection/benchmark_script.py b/core/benchmarks/object-detection/benchmark_script.py index cd794f64e..efffdb48a 100644 --- a/core/benchmarks/object-detection/benchmark_script.py +++ b/core/benchmarks/object-detection/benchmark_script.py @@ -21,6 +21,7 @@ Polygon, Prediction, Raster, + ValorDetectionManager, enums, evaluate_detection, ) @@ -235,11 +236,59 @@ def run_detailed_pr_curve_evaluation(groundtruths, predictions): enums.MetricType.mAR, enums.MetricType.mAPAveragedOverIOUs, enums.MetricType.PrecisionRecallCurve, + enums.MetricType.DetailedPrecisionRecallCurve, ], ) return evaluation +def run_base_evaluation_with_manager(groundtruths, predictions): + """Run a base evaluation (with no PR curves) using ValorDetectionManager.""" + manager = ValorDetectionManager() + manager.add_data(groundtruths=groundtruths, predictions=predictions) + return manager.evaluate() + + +def run_pr_curve_evaluation_with_manager(groundtruths, predictions): + """Run a base evaluation with PrecisionRecallCurve included using ValorDetectionManager.""" + manager = ValorDetectionManager( + metrics_to_return=[ + enums.MetricType.AP, + enums.MetricType.AR, + enums.MetricType.mAP, + enums.MetricType.APAveragedOverIOUs, + enums.MetricType.mAR, + enums.MetricType.mAPAveragedOverIOUs, + enums.MetricType.PrecisionRecallCurve, + ], + ) + + manager.add_data(groundtruths=groundtruths, predictions=predictions) + + return manager.evaluate() + + +def run_detailed_pr_curve_evaluation_with_manager(groundtruths, predictions): + """Run a base evaluation with PrecisionRecallCurve and DetailedPrecisionRecallCurve included using ValorDetectionManager.""" + + manager = ValorDetectionManager( + metrics_to_return=[ + enums.MetricType.AP, + enums.MetricType.AR, + enums.MetricType.mAP, + enums.MetricType.APAveragedOverIOUs, + enums.MetricType.mAR, + enums.MetricType.mAPAveragedOverIOUs, + enums.MetricType.PrecisionRecallCurve, + enums.MetricType.DetailedPrecisionRecallCurve, + ], + ) + + manager.add_data(groundtruths=groundtruths, predictions=predictions) + + return manager.evaluate() + + @dataclass class DataBenchmark: dtype: str @@ -364,9 +413,13 @@ def run_benchmarking_analysis( # run evaluations eval_pr = None eval_detail = None - eval_base = run_base_evaluation(groundtruths, predictions) + eval_base = run_base_evaluation_with_manager( + groundtruths, predictions + ) if compute_pr: - eval_pr = run_pr_curve_evaluation(groundtruths, predictions) + eval_pr = run_pr_curve_evaluation_with_manager( + groundtruths, predictions + ) if compute_detailed: eval_detail = run_detailed_pr_curve_evaluation( groundtruths, predictions @@ -422,11 +475,11 @@ def run_benchmarking_analysis( ], limits_to_test=[5000, 5000], ) - # run raster benchmark run_benchmarking_analysis( combinations=[ (AnnotationType.RASTER, AnnotationType.RASTER), ], - limits_to_test=[500, 500], + limits_to_test=[500], + compute_detailed=False, ) diff --git a/core/tests/functional-tests/test_detection.py b/core/tests/functional-tests/test_detection.py index af9684501..98b77485f 100644 --- a/core/tests/functional-tests/test_detection.py +++ b/core/tests/functional-tests/test_detection.py @@ -1223,6 +1223,16 @@ def test_evaluate_detection_functional_test_with_rasters( pr_metrics[0]["value"][value][threshold][metric] == expected_value ) + # test that we get a NotImplementedError if we try to calculate DetailedPRCurves with rasters + with pytest.raises(NotImplementedError): + evaluate_detection( + groundtruths=evaluate_detection_functional_test_groundtruths_with_rasters, + predictions=evaluate_detection_functional_test_predictions_with_rasters, + metrics_to_return=[ + enums.MetricType.DetailedPrecisionRecallCurve, + ], + ) + def test_evaluate_mixed_annotations( evaluate_mixed_annotations_inputs: tuple, diff --git a/core/tests/functional-tests/test_detection_manager.py b/core/tests/functional-tests/test_detection_manager.py index e9e625a09..c7599aa3d 100644 --- a/core/tests/functional-tests/test_detection_manager.py +++ b/core/tests/functional-tests/test_detection_manager.py @@ -1002,6 +1002,21 @@ def test_evaluate_detection_functional_test_with_rasters_with_ValorDetectionMana pr_metrics[0]["value"][value][threshold][metric] == expected_value ) + # test that we get a NotImplementedError if we try to calculate DetailedPRCurves with rasters + manager = managers.ValorDetectionManager( + metrics_to_return=[ + enums.MetricType.DetailedPrecisionRecallCurve, + ], + ) + + manager.add_data( + groundtruths=evaluate_detection_functional_test_groundtruths_with_rasters, + predictions=evaluate_detection_functional_test_predictions_with_rasters, + ) + + with pytest.raises(NotImplementedError): + manager.evaluate() + def test_evaluate_mixed_annotations_with_ValorDetectionManager( evaluate_mixed_annotations_inputs: tuple, diff --git a/core/valor_core/detection.py b/core/valor_core/detection.py index c28eaf133..e4fff23cc 100644 --- a/core/valor_core/detection.py +++ b/core/valor_core/detection.py @@ -99,41 +99,19 @@ def _calculate_iou( joint_df["iou_"] = 0 else: - iou_calculation_df = ( - joint_df.assign( - intersection=lambda chain_df: chain_df.apply( - lambda row: ( - 0 - if row["converted_geometry_pd"] is None - or row["converted_geometry_gt"] is None - else np.logical_and( - row["converted_geometry_pd"], - row["converted_geometry_gt"], - ).sum() - ), - axis=1, - ) - ) - .assign( - union_=lambda chain_df: chain_df.apply( - lambda row: ( - 0 - if row["converted_geometry_pd"] is None - or row["converted_geometry_gt"] is None - else np.sum(row["converted_geometry_gt"]) - + np.sum(row["converted_geometry_pd"]) - - row["intersection"] - ), - axis=1, - ) - ) - .assign( - iou_=lambda chain_df: chain_df["intersection"] - / chain_df["union_"] + intersection_ = joint_df.apply( + geometry.calculate_raster_intersection, + axis=1, + ) + union_ = ( + joint_df.apply( + geometry.calculate_raster_union, + axis=1, ) + - intersection_ ) - joint_df = joint_df.join(iou_calculation_df["iou_"]) + joint_df["iou_"] = intersection_ / union_ return joint_df @@ -643,6 +621,21 @@ def _calculate_detailed_pr_metrics( ) or (detailed_pr_joint_df is None): return [] + if _check_if_series_contains_masks( + detailed_pr_joint_df.loc[ + detailed_pr_joint_df["converted_geometry_pd"].notnull(), + "converted_geometry_pd", + ] + ) or _check_if_series_contains_masks( + detailed_pr_joint_df.loc[ + detailed_pr_joint_df["converted_geometry_pd"].notnull(), + "converted_geometry_pd", + ] + ): + raise NotImplementedError( + "DetailedPrecisionRecallCurves are not yet implemented when dealing with rasters." + ) + # add confidence_threshold to the dataframe and sort detailed_pr_calc_df = pd.concat( [ @@ -1045,11 +1038,12 @@ def _create_detailed_joint_df( on=["datum_id", "label_key"], how="outer", suffixes=("_gt", "_pd"), - ).assign( - is_label_match=lambda chain_df: ( - chain_df["label_id_pd"] == chain_df["label_id_gt"] - ) ) + + detailed_joint_df["is_label_match"] = ( + detailed_joint_df["label_id_pd"] == detailed_joint_df["label_id_gt"] + ) + detailed_joint_df = _calculate_iou(joint_df=detailed_joint_df) return detailed_joint_df diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index 86db91598..8f3017ac0 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -1,5 +1,6 @@ import numba import numpy as np +import pandas as pd import shapely.affinity from shapely.geometry import Polygon as ShapelyPolygon @@ -244,3 +245,42 @@ def is_rotated(bbox: list[tuple[float, float]]) -> bool: True if the bounding box is rotated, otherwise False. """ return not is_axis_aligned(bbox) and not is_skewed(bbox) + + +def calculate_raster_intersection(row: pd.Series) -> pd.Series: + """ + Calculate the raster intersection for a given row in a pandas DataFrame. This function is intended to be used with .apply. + + Parameters + ---------- + row : pd.Series + A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt". + + Returns + ---------- + pd.Series + A Series indicating the intersection of two masks. + """ + return np.logical_and( + row["converted_geometry_pd"], row["converted_geometry_gt"] + ).sum() + + +def calculate_raster_union(row: pd.Series) -> pd.Series: + """ + Calculate the raster union for a given row in a pandas DataFrame. This function is intended to be used with .apply. + + Parameters + ---------- + row : pd.Series + A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt". + + Returns + ---------- + pd.Series + A Series indicating the union of two masks. + """ + + return np.sum(row["converted_geometry_gt"]) + np.sum( + row["converted_geometry_pd"] + ) From 74819e7fd70f46ad9650609b4c920522a4f371af Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 11:22:31 -0600 Subject: [PATCH 2/8] update benchmark script --- core/benchmarks/object-detection/benchmark_script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/benchmarks/object-detection/benchmark_script.py b/core/benchmarks/object-detection/benchmark_script.py index efffdb48a..85d03d9fc 100644 --- a/core/benchmarks/object-detection/benchmark_script.py +++ b/core/benchmarks/object-detection/benchmark_script.py @@ -466,6 +466,7 @@ def run_benchmarking_analysis( (AnnotationType.BOX, AnnotationType.BOX), ], limits_to_test=[5000, 5000], + compute_detailed=False, ) # run polygon benchmark @@ -474,6 +475,7 @@ def run_benchmarking_analysis( (AnnotationType.POLYGON, AnnotationType.POLYGON), ], limits_to_test=[5000, 5000], + compute_detailed=False, ) # run raster benchmark run_benchmarking_analysis( From f8fda09935c1bfeea2f14ccf75ef812f2a8cd848 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 11:54:02 -0600 Subject: [PATCH 3/8] fix Charles' feedback --- core/benchmarks/object-detection/benchmark_script.py | 2 +- core/valor_core/detection.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/benchmarks/object-detection/benchmark_script.py b/core/benchmarks/object-detection/benchmark_script.py index 85d03d9fc..52e39b710 100644 --- a/core/benchmarks/object-detection/benchmark_script.py +++ b/core/benchmarks/object-detection/benchmark_script.py @@ -482,6 +482,6 @@ def run_benchmarking_analysis( combinations=[ (AnnotationType.RASTER, AnnotationType.RASTER), ], - limits_to_test=[500], + limits_to_test=[500, 500], compute_detailed=False, ) diff --git a/core/valor_core/detection.py b/core/valor_core/detection.py index e4fff23cc..6a8c19de2 100644 --- a/core/valor_core/detection.py +++ b/core/valor_core/detection.py @@ -623,8 +623,8 @@ def _calculate_detailed_pr_metrics( if _check_if_series_contains_masks( detailed_pr_joint_df.loc[ - detailed_pr_joint_df["converted_geometry_pd"].notnull(), - "converted_geometry_pd", + detailed_pr_joint_df["converted_geometry_gt"].notnull(), + "converted_geometry_gt", ] ) or _check_if_series_contains_masks( detailed_pr_joint_df.loc[ From 5be69152c29249a3604f6edc9612f963cdb2169c Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 12:02:57 -0600 Subject: [PATCH 4/8] test parallel=True --- core/valor_core/geometry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index 8f3017ac0..d338f2237 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -9,7 +9,7 @@ np.seterr(divide="ignore", invalid="ignore") -@numba.jit(nopython=True) +@numba.jit(nopython=True, parallel=True) def calculate_axis_aligned_bbox_intersection( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: @@ -45,7 +45,7 @@ def calculate_axis_aligned_bbox_intersection( return intersection_area -@numba.jit(nopython=True) +@numba.jit(nopython=True, parallel=True) def calculate_axis_aligned_bbox_union( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: @@ -79,7 +79,7 @@ def calculate_axis_aligned_bbox_union( return union_area -@numba.jit(nopython=True) +@numba.jit(nopython=True, parallel=True) def calculate_axis_aligned_bbox_iou( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: From f3808e67851d60f6c3326d0289c1373a6351fd25 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 12:14:35 -0600 Subject: [PATCH 5/8] remove parallelization; incorporate Charles' feedback --- core/valor_core/detection.py | 2 +- core/valor_core/geometry.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/valor_core/detection.py b/core/valor_core/detection.py index 6a8c19de2..15e8d8b02 100644 --- a/core/valor_core/detection.py +++ b/core/valor_core/detection.py @@ -105,7 +105,7 @@ def _calculate_iou( ) union_ = ( joint_df.apply( - geometry.calculate_raster_union, + geometry.calculate_raster_sum_pixels_of_two_images, axis=1, ) - intersection_ diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index d338f2237..706e20e56 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -9,7 +9,7 @@ np.seterr(divide="ignore", invalid="ignore") -@numba.jit(nopython=True, parallel=True) +@numba.jit(nopython=True) def calculate_axis_aligned_bbox_intersection( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: @@ -45,7 +45,7 @@ def calculate_axis_aligned_bbox_intersection( return intersection_area -@numba.jit(nopython=True, parallel=True) +@numba.jit(nopython=True) def calculate_axis_aligned_bbox_union( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: @@ -79,7 +79,7 @@ def calculate_axis_aligned_bbox_union( return union_area -@numba.jit(nopython=True, parallel=True) +@numba.jit(nopython=True) def calculate_axis_aligned_bbox_iou( bbox1: np.ndarray, bbox2: np.ndarray ) -> float: @@ -266,9 +266,9 @@ def calculate_raster_intersection(row: pd.Series) -> pd.Series: ).sum() -def calculate_raster_union(row: pd.Series) -> pd.Series: +def calculate_raster_sum_pixels_of_two_images(row: pd.Series) -> pd.Series: """ - Calculate the raster union for a given row in a pandas DataFrame. This function is intended to be used with .apply. + Calculate the sum of pixels across two images for a given row in a pandas DataFrame. When we subtract the intersection series from this output, we expect to get back the union for all sets of images. This function is intended to be used with .apply. Parameters ---------- @@ -278,7 +278,7 @@ def calculate_raster_union(row: pd.Series) -> pd.Series: Returns ---------- pd.Series - A Series indicating the union of two masks. + A Series indicating the sum of pixels across two masks. """ return np.sum(row["converted_geometry_gt"]) + np.sum( From 1c1f7c2c744d2c1460919a86a8f5750a0abf1619 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 12:55:55 -0600 Subject: [PATCH 6/8] change function --- core/valor_core/geometry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index 706e20e56..78ffb6558 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -281,6 +281,6 @@ def calculate_raster_sum_pixels_of_two_images(row: pd.Series) -> pd.Series: A Series indicating the sum of pixels across two masks. """ - return np.sum(row["converted_geometry_gt"]) + np.sum( - row["converted_geometry_pd"] - ) + return np.logical_or( + row["converted_geometry_pd"], row["converted_geometry_gt"] + ).sum() From 509b735293b559773cb1ca18ed3a6959a69cb562 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 13:09:15 -0600 Subject: [PATCH 7/8] fix intersection --- core/valor_core/detection.py | 9 +++------ core/valor_core/geometry.py | 6 +++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/core/valor_core/detection.py b/core/valor_core/detection.py index 15e8d8b02..020b3a854 100644 --- a/core/valor_core/detection.py +++ b/core/valor_core/detection.py @@ -103,12 +103,9 @@ def _calculate_iou( geometry.calculate_raster_intersection, axis=1, ) - union_ = ( - joint_df.apply( - geometry.calculate_raster_sum_pixels_of_two_images, - axis=1, - ) - - intersection_ + union_ = joint_df.apply( + geometry.calculate_raster_union, + axis=1, ) joint_df["iou_"] = intersection_ / union_ diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index 78ffb6558..8ba3055f1 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -266,9 +266,9 @@ def calculate_raster_intersection(row: pd.Series) -> pd.Series: ).sum() -def calculate_raster_sum_pixels_of_two_images(row: pd.Series) -> pd.Series: +def calculate_raster_union(row: pd.Series) -> pd.Series: """ - Calculate the sum of pixels across two images for a given row in a pandas DataFrame. When we subtract the intersection series from this output, we expect to get back the union for all sets of images. This function is intended to be used with .apply. + Calculate the union across two rasters for a given row in a pandas DataFrame. This function is intended to be used with .apply. Parameters ---------- @@ -278,7 +278,7 @@ def calculate_raster_sum_pixels_of_two_images(row: pd.Series) -> pd.Series: Returns ---------- pd.Series - A Series indicating the sum of pixels across two masks. + A Series indicating the union of two masks. """ return np.logical_or( From c752d7c4e898a3452ffbca2b31fdb49fe6bf2e59 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 28 Aug 2024 14:57:09 -0600 Subject: [PATCH 8/8] add raster unit tests --- core/tests/unit-tests/test_geometry.py | 237 +++++++++++++++++++++++++ core/valor_core/detection.py | 13 +- core/valor_core/geometry.py | 43 +++-- 3 files changed, 261 insertions(+), 32 deletions(-) diff --git a/core/tests/unit-tests/test_geometry.py b/core/tests/unit-tests/test_geometry.py index 558912d19..479f9f1c4 100644 --- a/core/tests/unit-tests/test_geometry.py +++ b/core/tests/unit-tests/test_geometry.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import pytest from valor_core import geometry from valor_core.schemas import ( @@ -1041,6 +1042,242 @@ def test_calculate_iou(): assert expected == round(iou, 4) +def test_calculate_raster_iou(): + filled_8x8 = np.full((8, 8), True) + filled_10x10 = (np.full((10, 10), True),) + + series1 = pd.Series( + [ + filled_10x10, + filled_10x10, + [ + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + [True, True, True, True, True, False, False, False], + ], + filled_8x8, + filled_8x8, + ] + ) + + series2 = pd.Series( + [ + [ + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + [True, True, True, True, True, True, True, True, True, True], + ], + [ + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + [ + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + ], + ], + [ + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + ], + [ + [False, False, False, False, True, True, True, True], + [False, False, False, False, True, True, True, True], + [False, False, False, False, True, True, True, True], + [False, False, False, False, True, True, True, True], + [False, False, False, False, False, False, False, False], + [False, False, False, False, False, False, False, False], + [False, False, False, False, False, False, False, False], + [False, False, False, False, False, False, False, False], + ], + [ + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [False, False, False, False, False, True, True, True], + [True, True, True, False, False, True, True, True], + [True, True, True, False, False, True, True, True], + [True, True, True, False, False, True, True, True], + [True, True, True, False, False, True, True, True], + ], + ] + ) + + result = geometry.calculate_raster_ious(series1, series2) + assert (result == [1, 0.5, 0, 0.25, 36 / 64]).all() + + # check that we throw an error if the series aren't the same length + series1 = pd.Series( + [ + filled_10x10, + filled_10x10, + ] + ) + + series2 = pd.Series( + [ + filled_10x10, + filled_10x10, + filled_10x10, + ] + ) + with pytest.raises(ValueError) as e: + geometry.calculate_raster_ious(series1, series2) + assert ( + "Series of rasters must be the same length to calculate IOUs." + in str(e) + ) + + # check that we don't compare rasters that aren't the same size + series1 = pd.Series( + [ + filled_10x10, + filled_10x10, + ] + ) + + series2 = pd.Series( + [ + filled_10x10, + filled_8x8, + ] + ) + with pytest.raises(ValueError) as e: + geometry.calculate_raster_ious(series1, series2) + assert "operands could not be broadcast together with shapes" in str(e) + + def test_is_axis_aligned(box_points, skewed_box_points, rotated_box_points): tests = [ { diff --git a/core/valor_core/detection.py b/core/valor_core/detection.py index 020b3a854..a00ff2265 100644 --- a/core/valor_core/detection.py +++ b/core/valor_core/detection.py @@ -99,16 +99,11 @@ def _calculate_iou( joint_df["iou_"] = 0 else: - intersection_ = joint_df.apply( - geometry.calculate_raster_intersection, - axis=1, - ) - union_ = joint_df.apply( - geometry.calculate_raster_union, - axis=1, - ) - joint_df["iou_"] = intersection_ / union_ + joint_df["iou_"] = geometry.calculate_raster_ious( + joint_df["converted_geometry_gt"], + joint_df["converted_geometry_pd"], + ) return joint_df diff --git a/core/valor_core/geometry.py b/core/valor_core/geometry.py index 8ba3055f1..797b321fd 100644 --- a/core/valor_core/geometry.py +++ b/core/valor_core/geometry.py @@ -247,40 +247,37 @@ def is_rotated(bbox: list[tuple[float, float]]) -> bool: return not is_axis_aligned(bbox) and not is_skewed(bbox) -def calculate_raster_intersection(row: pd.Series) -> pd.Series: +def calculate_raster_ious(series1: pd.Series, series2: pd.Series) -> pd.Series: """ - Calculate the raster intersection for a given row in a pandas DataFrame. This function is intended to be used with .apply. + Calculate the IOUs between two series of rasters. Parameters ---------- - row : pd.Series - A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt". + series1 : pd.Series + The first series of rasters. + series2: pd.Series + The second series of rasters. Returns ---------- pd.Series - A Series indicating the intersection of two masks. + A Series of IOUs. """ - return np.logical_and( - row["converted_geometry_pd"], row["converted_geometry_gt"] - ).sum() + if len(series1) != len(series2): + raise ValueError( + "Series of rasters must be the same length to calculate IOUs." + ) -def calculate_raster_union(row: pd.Series) -> pd.Series: - """ - Calculate the union across two rasters for a given row in a pandas DataFrame. This function is intended to be used with .apply. + intersection_ = pd.Series( + [np.logical_and(x, y).sum() for x, y in zip(series1, series2)] + ) - Parameters - ---------- - row : pd.Series - A row of a pandas.DataFrame containing two masks in the columns "converted_geometry_pd" and "converted_geometry_gt". + union_ = pd.Series( + [np.logical_or(x, y).sum() for x, y in zip(series1, series2)] + ) - Returns - ---------- - pd.Series - A Series indicating the union of two masks. - """ + if (intersection_ > union_).any(): + raise ValueError("Intersection can't be greater than union.") - return np.logical_or( - row["converted_geometry_pd"], row["converted_geometry_gt"] - ).sum() + return intersection_ / union_