Fix Classification Performance

api/tests/functional-tests/backend/metrics/test_classification.py

@@ -219,12 +219,12 @@ def test_compute_confusion_matrix_at_grouper_key(
         models.Annotation.datum_id.label("datum_id"),
         filters=gFilter,
         label_source=models.GroundTruth,
-    ).alias()
+    ).cte()
     predictions = generate_select(
         models.Prediction,
         filters=pFilter,
         label_source=models.Prediction,
-    ).alias()
+    ).cte()
 
     cm = _compute_confusion_matrix_at_grouper_key(
         db=db,
@@ -294,12 +294,12 @@ def test_compute_confusion_matrix_at_grouper_key(
         models.Annotation.datum_id.label("datum_id"),
         filters=gFilter,
         label_source=models.GroundTruth,
-    ).alias()
+    ).cte()
     predictions = generate_select(
         models.Prediction,
         filters=pFilter,
         label_source=models.Prediction,
-    ).alias()
+    ).cte()
 
     cm = _compute_confusion_matrix_at_grouper_key(
         db=db,
@@ -445,12 +445,12 @@ def test_compute_confusion_matrix_at_grouper_key_and_filter(
         models.Annotation.datum_id.label("datum_id"),
         filters=gFilter,
         label_source=models.GroundTruth,
-    ).alias()
+    ).cte()
     predictions = generate_select(
         models.Prediction,
         filters=pFilter,
         label_source=models.Prediction,
-    ).alias()
+    ).cte()
 
     cm = _compute_confusion_matrix_at_grouper_key(
         db,
@@ -595,12 +595,12 @@ def test_compute_confusion_matrix_at_grouper_key_using_label_map(
         models.Annotation.datum_id.label("datum_id"),
         filters=gFilter,
         label_source=models.GroundTruth,
-    ).alias()
+    ).cte()
     predictions = generate_select(
         models.Prediction,
         filters=pFilter,
         label_source=models.Prediction,
-    ).alias()
+    ).cte()
 
     cm = _compute_confusion_matrix_at_grouper_key(
         db,
@@ -1256,32 +1256,44 @@ def test__compute_curves(
         models.Dataset.name.label("dataset_name"),
         filters=gFilter,
         label_source=models.GroundTruth,
-    ).alias()
+    ).cte()
     predictions = generate_select(
         models.Prediction,
+        models.Annotation.datum_id.label("datum_id"),
         models.Dataset.name.label("dataset_name"),
         filters=pFilter,
         label_source=models.Prediction,
-    ).alias()
+    ).cte()
 
     # calculate the number of unique datums
     # used to determine the number of true negatives
 
     gt_datums = generate_query(
+        models.Datum.id,
         models.Dataset.name,
         models.Datum.uid,
         db=db,
         filters=groundtruth_filter,
         label_source=models.GroundTruth,
     ).all()
     pd_datums = generate_query(
+        models.Datum.id,
         models.Dataset.name,
         models.Datum.uid,
         db=db,
         filters=prediction_filter,
         label_source=models.Prediction,
     ).all()
-    unique_datums = set(pd_datums + gt_datums)
+    unique_datums = {
+        datum_id: (dataset_name, datum_uid)
+        for datum_id, dataset_name, datum_uid in gt_datums
+    }
+    unique_datums.update(
+        {
+            datum_id: (dataset_name, datum_uid)
+            for datum_id, dataset_name, datum_uid in pd_datums
+        }
+    )
 
     curves = _compute_curves(
         db=db,
@@ -1368,8 +1380,8 @@ def test__compute_curves(
         },
         ("dog", 0.05, "tn"): {"all": 1, "total": 1},
         ("dog", 0.8, "fn"): {
-            "missed_detections": 1,
-            "misclassifications": 1,
+            "missed_detections": 0,
+            "misclassifications": 2,
             "total": 2,
         },
         # cat

api/tests/functional-tests/crud/test_read.py

@@ -329,7 +329,8 @@ def test_get_dataset_summary(
         enums.TaskType.CLASSIFICATION,
         enums.TaskType.EMPTY,
     }
-    assert summary.datum_metadata == [
+
+    expected_datum_metadata = [
         {
             "width": 32,
             "height": 80,
@@ -339,10 +340,19 @@ def test_get_dataset_summary(
             "height": 100,
         },
     ]
-    assert summary.annotation_metadata == [
+    for item in summary.datum_metadata:
+        assert item in expected_datum_metadata
+    for item in expected_datum_metadata:
+        assert item in summary.datum_metadata
+
+    expected_annotation_metadata = [
         {"int_key": 1},
         {
             "string_key": "string_val",
             "int_key": 1,
         },
     ]
+    for item in summary.annotation_metadata:
+        assert item in expected_annotation_metadata
+    for item in expected_annotation_metadata:
+        assert item in summary.annotation_metadata

api/valor_api/backend/core/evaluation.py

@@ -294,9 +294,9 @@ def _validate_evaluation_filter(
     if parameters.task_type == enums.TaskType.CLASSIFICATION:
         core.validate_matching_label_keys(
             db=db,
+            dataset_names=evaluation.dataset_names,
+            model_name=evaluation.model_name,
             label_map=parameters.label_map,
-            groundtruth_filter=groundtruth_filter,
-            prediction_filter=predictions_filter,
         )
 
 

api/valor_api/backend/core/label.py

@@ -7,17 +7,17 @@
 
 from valor_api import api_utils, schemas
 from valor_api.backend import models
-from valor_api.backend.query import generate_query, generate_select
+from valor_api.backend.query import generate_query
 from valor_api.backend.query.types import TableTypeAlias
 
 LabelMapType = list[list[list[str]]]
 
 
 def validate_matching_label_keys(
     db: Session,
+    dataset_names: list[str],
+    model_name: str,
     label_map: LabelMapType | None,
-    prediction_filter: schemas.Filter,
-    groundtruth_filter: schemas.Filter,
 ) -> None:
     """
     Validates that every datum has the same set of label keys for both ground truths and predictions. This check is only needed for classification tasks.
@@ -26,71 +26,95 @@ def validate_matching_label_keys(
     ----------
     db : Session
         The database Session to query against.
-    prediction_filter : schemas.Filter
-        The filter to be used to query predictions.
-    groundtruth_filter : schemas.Filter
-        The filter to be used to query groundtruths.
+    dataset_names : list[str]
+        The list of required datasets by name.
+    model_name : str
+        The required model by name.
     label_map: LabelMapType, optional
         Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 
-
     Raises
     -------
     ValueError
         If the distinct ground truth label keys don't match the distinct prediction label keys for any datum.
     """
 
-    gts = generate_select(
-        models.Annotation.datum_id.label("datum_id"),
-        models.Label.key.label("label_key"),
-        models.Label.value.label("label_value"),
-        filters=groundtruth_filter,
-        label_source=models.GroundTruth,
-    ).alias()
-
-    gt_label_keys_by_datum = (
+    gt_labels_by_datum = (
         select(
-            gts.c.datum_id,
-            func.array_agg(gts.c.label_key + ", " + gts.c.label_value).label(
+            models.Datum.id.label("datum_id"),
+            func.array_agg(models.Label.key + ", " + models.Label.value).label(
                 "gt_labels"
             ),
         )
-        .select_from(gts)
-        .group_by(gts.c.datum_id)
+        .select_from(models.Datum)
+        .join(
+            models.Dataset,
+            and_(
+                models.Dataset.id == models.Datum.dataset_id,
+                models.Dataset.name.in_(dataset_names),
+            ),
+        )
+        .join(
+            models.Annotation,
+            and_(
+                models.Annotation.datum_id == models.Datum.id,
+                models.Annotation.model_id.is_(None),
+            ),
+        )
+        .join(
+            models.GroundTruth,
+            models.GroundTruth.annotation_id == models.Annotation.id,
+        )
+        .join(models.Label, models.Label.id == models.GroundTruth.label_id)
+        .group_by(models.Datum.id)
         .subquery()
     )
 
-    preds = generate_select(
-        models.Annotation.datum_id.label("datum_id"),
-        models.Label.key.label("label_key"),
-        models.Label.value.label("label_value"),
-        filters=prediction_filter,
-        label_source=models.Prediction,
-    ).alias()
-
-    preds_label_keys_by_datum = (
+    pred_labels_by_datum = (
         select(
-            preds.c.datum_id,
-            func.array_agg(
-                preds.c.label_key + ", " + preds.c.label_value
-            ).label("pred_labels"),
+            models.Datum.id.label("datum_id"),
+            func.array_agg(models.Label.key + ", " + models.Label.value).label(
+                "pred_labels"
+            ),
+        )
+        .select_from(models.Datum)
+        .join(
+            models.Dataset,
+            and_(
+                models.Dataset.id == models.Datum.dataset_id,
+                models.Dataset.name.in_(dataset_names),
+            ),
+        )
+        .join(
+            models.Annotation,
+            models.Annotation.datum_id == models.Datum.id,
+        )
+        .join(
+            models.Model,
+            and_(
+                models.Model.id == models.Annotation.model_id,
+                models.Model.name == model_name,
+            ),
+        )
+        .join(
+            models.Prediction,
+            models.Prediction.annotation_id == models.Annotation.id,
         )
-        .select_from(preds)
-        .group_by(preds.c.datum_id)
+        .join(models.Label, models.Label.id == models.Prediction.label_id)
+        .group_by(models.Datum.id)
         .subquery()
     )
 
     joined = (
         select(
-            preds_label_keys_by_datum.c.datum_id,
-            preds_label_keys_by_datum.c.pred_labels,
-            gt_label_keys_by_datum.c.gt_labels,
+            pred_labels_by_datum.c.datum_id,
+            pred_labels_by_datum.c.pred_labels,
+            gt_labels_by_datum.c.gt_labels,
         )
-        .select_from(preds_label_keys_by_datum)
+        .select_from(gt_labels_by_datum)
         .join(
-            gt_label_keys_by_datum,
-            gt_label_keys_by_datum.c.datum_id
-            == preds_label_keys_by_datum.c.datum_id,
+            pred_labels_by_datum,
+            pred_labels_by_datum.c.datum_id == gt_labels_by_datum.c.datum_id,
         )
         .subquery()
     )

api/valor_api/backend/core/prediction.py

@@ -9,20 +9,30 @@
 def _check_if_datum_has_prediction(
     db: Session, datum: schemas.Datum, model_name: str, dataset_name: str
 ) -> None:
+    """Checks to see if datum has existing annotations."""
     if db.query(
         select(models.Annotation.id)
-        .join(models.Model)
-        .join(models.Datum)
-        .join(models.Dataset)
-        .where(
+        .select_from(models.Annotation)
+        .join(
+            models.Model,
             and_(
-                models.Dataset.name == dataset_name,
-                models.Datum.dataset_id == models.Dataset.id,
-                models.Datum.uid == datum.uid,
+                models.Model.id == models.Annotation.model_id,
                 models.Model.name == model_name,
-                models.Annotation.datum_id == models.Datum.id,
-                models.Annotation.model_id == models.Model.id,
-            )
+            ),
+        )
+        .join(
+            models.Datum,
+            and_(
+                models.Datum.id == models.Annotation.datum_id,
+                models.Datum.uid == datum.uid,
+            ),
+        )
+        .join(
+            models.Dataset,
+            and_(
+                models.Dataset.id == models.Datum.dataset_id,
+                models.Dataset.name == dataset_name,
+            ),
         )
         .subquery()
     ).all():
@@ -119,15 +129,14 @@ def create_predictions(
         for i, annotation in enumerate(prediction.annotations):
             for label in annotation.labels:
                 prediction_mappings.append(
-                    {
-                        "annotation_id": annotation_ids_per_prediction[i],
-                        "label_id": label_dict[(label.key, label.value)],
-                        "score": label.score,
-                    }
+                    models.Prediction(
+                        annotation_id=annotation_ids_per_prediction[i],
+                        label_id=label_dict[(label.key, label.value)],
+                        score=label.score,
+                    )
                 )
-
     try:
-        db.bulk_insert_mappings(models.Prediction, prediction_mappings)
+        db.add_all(prediction_mappings)
         db.commit()
     except IntegrityError as e:
         db.rollback()