Object Detection Benchmarking (#704)

Striveworks · Aug 16, 2024 · 40f7b3b · 40f7b3b
1 parent 60622a3
commit 40f7b3b
Show file tree

Hide file tree

Showing 11 changed files with 6,300 additions and 1,702 deletions.
diff --git a/api/tests/functional-tests/crud/test_create_delete.py b/api/tests/functional-tests/crud/test_create_delete.py
@@ -1064,15 +1064,11 @@ def test_gt_seg_as_mask_or_polys(
     assert len(segs.annotations) == 2
 
     assert segs.annotations[0].raster and segs.annotations[1].raster
-    decoded_mask0 = np.array(
-        _bytes_to_pil(b64decode(segs.annotations[0].raster.mask))
-    )
+    decoded_mask0 = segs.annotations[0].raster.array
     assert decoded_mask0.shape == mask.shape
     np.testing.assert_equal(decoded_mask0, mask)
 
-    decoded_mask1 = np.array(
-        _bytes_to_pil(b64decode(segs.annotations[1].raster.mask))
-    )
+    decoded_mask1 = segs.annotations[1].raster.array
     assert decoded_mask1.shape == mask.shape
     np.testing.assert_equal(decoded_mask1, mask)
 

diff --git a/api/valor_api/backend/metrics/detection.py b/api/valor_api/backend/metrics/detection.py
@@ -7,7 +7,7 @@
 from typing import Sequence, Tuple
 
 from geoalchemy2 import functions as gfunc
-from sqlalchemy import CTE, and_, func, or_, select
+from sqlalchemy import CTE, and_, case, func, or_, select
 from sqlalchemy.orm import Session, aliased
 
 from valor_api import enums, schemas
@@ -458,10 +458,14 @@ def _compute_detailed_curves(
                         if misclassification_detected:
                             fn["misclassifications"].append(
                                 (dataset_name, datum_uid, gt_geojson)
+                                if gt_geojson is not None
+                                else (dataset_name, datum_uid)
                             )
                         else:
                             fn["no_predictions"].append(
                                 (dataset_name, datum_uid, gt_geojson)
+                                if gt_geojson is not None
+                                else (dataset_name, datum_uid)
                             )
 
             if label_id in predictions_per_label:
@@ -501,10 +505,14 @@ def _compute_detailed_curves(
                         if misclassification_detected:
                             fp["misclassifications"].append(
                                 (dataset_name, datum_uid, pd_geojson)
+                                if pd_geojson is not None
+                                else (dataset_name, datum_uid)
                             )
                         elif hallucination_detected:
                             fp["hallucinations"].append(
                                 (dataset_name, datum_uid, pd_geojson)
+                                if pd_geojson is not None
+                                else (dataset_name, datum_uid)
                             )
 
             # calculate metrics
@@ -1158,14 +1166,22 @@ def _annotation_type_to_column(
             select(
                 gt_pd_counts.c.gt_annotation_id,
                 gt_pd_counts.c.pd_annotation_id,
-                func.coalesce(
-                    gt_pd_counts.c.intersection
-                    / (
+                case(
+                    (
                         gt_pd_counts.c.gt_count
                         + gt_pd_counts.c.pd_count
                         - gt_pd_counts.c.intersection
+                        == 0,
+                        0,
+                    ),
+                    else_=(
+                        gt_pd_counts.c.intersection
+                        / (
+                            gt_pd_counts.c.gt_count
+                            + gt_pd_counts.c.pd_count
+                            - gt_pd_counts.c.intersection
+                        )
                     ),
-                    0,
                 ).label("iou"),
             )
             .select_from(gt_pd_counts)
@@ -1183,7 +1199,10 @@ def _annotation_type_to_column(
             select(
                 gt_pd_pairs.c.gt_annotation_id,
                 gt_pd_pairs.c.pd_annotation_id,
-                iou_computation.label("iou"),
+                case(
+                    (gfunc.ST_Area(gunion) == 0, 0),
+                    else_=iou_computation,
+                ).label("iou"),
             )
             .select_from(gt_pd_pairs)
             .join(
@@ -1209,10 +1228,7 @@ def _annotation_type_to_column(
             gt.c.label_id.label("gt_label_id"),
             pd.c.label_id.label("pd_label_id"),
             pd.c.score.label("score"),
-            func.coalesce(
-                gt_pd_ious.c.iou,
-                0,
-            ).label("iou"),
+            gt_pd_ious.c.iou,
             gt.c.geojson.label("gt_geojson"),
         )
         .select_from(pd)
@@ -1576,14 +1592,22 @@ def _annotation_type_to_column(
             select(
                 gt_pd_counts.c.gt_annotation_id,
                 gt_pd_counts.c.pd_annotation_id,
-                func.coalesce(
-                    gt_pd_counts.c.intersection
-                    / (
+                case(
+                    (
                         gt_pd_counts.c.gt_count
                         + gt_pd_counts.c.pd_count
                         - gt_pd_counts.c.intersection
+                        == 0,
+                        0,
+                    ),
+                    else_=(
+                        gt_pd_counts.c.intersection
+                        / (
+                            gt_pd_counts.c.gt_count
+                            + gt_pd_counts.c.pd_count
+                            - gt_pd_counts.c.intersection
+                        )
                     ),
-                    0,
                 ).label("iou"),
             )
             .select_from(gt_pd_counts)
@@ -1601,7 +1625,10 @@ def _annotation_type_to_column(
             select(
                 gt_pd_pairs.c.gt_annotation_id,
                 gt_pd_pairs.c.pd_annotation_id,
-                iou_computation.label("iou"),
+                case(
+                    (gfunc.ST_Area(gunion) == 0, 0),
+                    else_=iou_computation,
+                ).label("iou"),
             )
             .select_from(gt_pd_pairs)
             .join(
@@ -1627,10 +1654,7 @@ def _annotation_type_to_column(
             gt.c.label_id.label("gt_label_id"),
             pd.c.label_id.label("pd_label_id"),
             pd.c.score.label("score"),
-            func.coalesce(
-                gt_pd_ious.c.iou,
-                0,
-            ).label("iou"),
+            gt_pd_ious.c.iou,
             gt.c.geojson.label("gt_geojson"),
             (gt.c.label_id == pd.c.label_id).label("is_match"),
         )

diff --git a/api/valor_api/schemas/geometry.py b/api/valor_api/schemas/geometry.py
@@ -11,7 +11,6 @@
     ST_GeomFromText,
     ST_MakeEmptyRaster,
     ST_MapAlgebra,
-    ST_SnapToGrid,
 )
 from pydantic import (
     BaseModel,
@@ -1028,8 +1027,8 @@ def to_psql(self) -> ScalarSelect | bytes:
         if self.geometry:
             empty_raster = ST_AddBand(
                 ST_MakeEmptyRaster(
-                    self.width,
-                    self.height,
+                    self.width,  # width
+                    self.height,  # height
                     0,  # upperleftx
                     0,  # upperlefty
                     1,  # scalex
@@ -1038,23 +1037,18 @@ def to_psql(self) -> ScalarSelect | bytes:
                     0,  # skewy
                     0,  # srid
                 ),
-                "1BB",
-            )
-            geom_raster = ST_AsRaster(
-                ST_SnapToGrid(
-                    ST_GeomFromText(self.geometry.to_wkt()),
-                    1.0,
-                ),
-                1.0,  # scalex
-                1.0,  # scaley
                 "1BB",  # pixeltype
-                1,  # value
-                0,  # nodataval
             )
             return select(
                 ST_MapAlgebra(
                     empty_raster,
-                    geom_raster,
+                    ST_AsRaster(
+                        ST_GeomFromText(self.geometry.to_wkt()),
+                        empty_raster,
+                        "1BB",
+                        1,
+                        0,
+                    ),
                     "[rast2]",
                     "1BB",
                     "UNION",

diff --git a/client/unit-tests/schemas/test_geojson.py b/client/unit-tests/schemas/test_geojson.py
@@ -105,7 +105,7 @@ def test_multipolygon():
         MultiPolygon([coords])  # type: ignore - testing
     with pytest.raises(TypeError):
         MultiPolygon([[coords], 123])  # type: ignore - testing
-    with pytest.raises(TypeError):
+    with pytest.raises(ValueError):
         MultiPolygon([[[coords]]])  # type: ignore - testing
 
 

diff --git a/client/valor/schemas/symbolic/types.py b/client/valor/schemas/symbolic/types.py
@@ -769,12 +769,14 @@ def __init__(
 
     @classmethod
     def __validate__(cls, value: typing.Any):
-        if not isinstance(value, tuple):
+        if not isinstance(value, (tuple, list)):
             raise TypeError(
                 f"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'"
             )
         elif len(value) != 2:
-            raise ValueError("")
+            raise ValueError(
+                "A point should contain only two x-y coordinates."
+            )
         for item in value:
             if not isinstance(item, (int, float, np.floating)):
                 raise TypeError(