diff --git a/api/tests/functional-tests/backend/metrics/test_detection.py b/api/tests/functional-tests/backend/metrics/test_detection.py index f73b5d020..7f4e4d3d1 100644 --- a/api/tests/functional-tests/backend/metrics/test_detection.py +++ b/api/tests/functional-tests/backend/metrics/test_detection.py @@ -1,13 +1,16 @@ +import numpy as np import pytest from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from valor_api import crud, enums, schemas +from valor_api.backend import core from valor_api.backend.metrics.detection import ( RankedPair, _compute_detailed_curves, _compute_detection_metrics, _compute_detection_metrics_with_detailed_precision_recall_curve, + _convert_annotations_to_common_type, compute_detection_metrics, ) from valor_api.backend.models import ( @@ -2276,3 +2279,147 @@ def test_detection_exceptions(db: Session): # show that no errors raised compute_detection_metrics(db=db, evaluation_id=evaluation_id) + + +def test__convert_annotations_to_common_type(db: Session): + + dataset_name = "dataset" + model_name = "model" + + xmin, xmax, ymin, ymax = 11, 45, 37, 102 + h, w = 150, 200 + mask = np.zeros((h, w), dtype=bool) + mask[ymin:ymax, xmin:xmax] = True + + pts = [ + (xmin, ymin), + (xmin, ymax), + (xmax, ymax), + (xmax, ymin), + (xmin, ymin), + ] + poly = schemas.Polygon(value=[pts]) + raster = schemas.Raster.from_numpy(mask) + box = schemas.Box.from_extrema(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax) + datum = schemas.Datum(uid="123") + + gt_box = schemas.GroundTruth( + datum=datum, + dataset_name=dataset_name, + annotations=[ + schemas.Annotation( + bounding_box=box, + labels=[schemas.Label(key="box", value="value")], + is_instance=True, + ) + ], + ) + gt_polygon = schemas.GroundTruth( + datum=datum, + dataset_name=dataset_name, + annotations=[ + schemas.Annotation( + polygon=poly, + labels=[schemas.Label(key="polygon", value="value")], + is_instance=True, + ) + ], + ) + gt_raster = schemas.GroundTruth( + datum=datum, + dataset_name=dataset_name, + annotations=[ + schemas.Annotation( + raster=raster, + labels=[schemas.Label(key="raster", value="value")], + is_instance=True, + ) + ], + ) + + pd_box = schemas.Prediction( + datum=datum, + dataset_name=dataset_name, + model_name=model_name, + annotations=[ + schemas.Annotation( + bounding_box=box, + labels=[schemas.Label(key="box", value="value", score=0.88)], + is_instance=True, + ) + ], + ) + pd_polygon = schemas.Prediction( + datum=datum, + dataset_name=dataset_name, + model_name=model_name, + annotations=[ + schemas.Annotation( + polygon=poly, + labels=[ + schemas.Label(key="polygon", value="value", score=0.89) + ], + is_instance=True, + ) + ], + ) + pd_raster = schemas.Prediction( + datum=datum, + dataset_name=dataset_name, + model_name=model_name, + annotations=[ + schemas.Annotation( + raster=raster, + labels=[schemas.Label(key="raster", value="value", score=0.9)], + is_instance=True, + ) + ], + ) + + gts = [ + (enums.AnnotationType.BOX, gt_box), + (enums.AnnotationType.POLYGON, gt_polygon), + (enums.AnnotationType.RASTER, gt_raster), + ] + pds = [ + (enums.AnnotationType.BOX, pd_box), + (enums.AnnotationType.POLYGON, pd_polygon), + (enums.AnnotationType.RASTER, pd_raster), + ] + + for gt_type, gt in gts: + for pd_type, pd in pds: + crud.create_dataset( + db=db, dataset=schemas.Dataset(name=dataset_name) + ) + crud.create_groundtruths(db=db, groundtruths=[gt]) + crud.finalize(db=db, dataset_name="dataset") + crud.create_model(db=db, model=schemas.Model(name=model_name)) + crud.create_predictions(db=db, predictions=[pd]) + + dataset = core.fetch_dataset(db=db, name=dataset_name) + model = core.fetch_model(db=db, name=model_name) + + for target_type in [ + enums.AnnotationType.RASTER, + enums.AnnotationType.POLYGON, + enums.AnnotationType.BOX, + ]: + if min(gt_type, pd_type) >= target_type: + _convert_annotations_to_common_type( + db=db, + datasets=[dataset], + model=model, + target_type=target_type, + ) + else: + with pytest.raises(ValueError): + _convert_annotations_to_common_type( + db=db, + datasets=[dataset], + model=model, + target_type=target_type, + ) + + crud.delete(db=db, dataset_name=dataset_name) + crud.delete(db=db, model_name=model_name) diff --git a/api/valor_api/backend/metrics/detection.py b/api/valor_api/backend/metrics/detection.py index 87a965f21..f20df6ed7 100644 --- a/api/valor_api/backend/metrics/detection.py +++ b/api/valor_api/backend/metrics/detection.py @@ -788,7 +788,9 @@ def _convert_annotations_to_common_type( db=db, dataset=dataset, task_type=enums.TaskType.OBJECT_DETECTION ) if target_type > source_type: - continue + raise ValueError( + f"Cannot convert dataset {dataset.name} annotations from {source_type} to {target_type}." + ) core.convert_geometry( db=db, dataset=dataset, @@ -803,6 +805,10 @@ def _convert_annotations_to_common_type( model=model, task_type=enums.TaskType.OBJECT_DETECTION, ) + if target_type > source_type: + raise ValueError( + f"Cannot convert model {model.name} annotations from {source_type} to {target_type}." + ) core.convert_geometry( db=db, dataset=dataset, diff --git a/integration_tests/client/metrics/test_detection.py b/integration_tests/client/metrics/test_detection.py index 9a5e8df7f..cab317477 100644 --- a/integration_tests/client/metrics/test_detection.py +++ b/integration_tests/client/metrics/test_detection.py @@ -3454,37 +3454,23 @@ def test_evaluate_mixed_annotations( for m in expected: assert m in eval_job.metrics - eval_job_raster = model.evaluate_detection( + eval_job_box = model.evaluate_detection( [dset_box, dset_polygon, dset_raster], iou_thresholds_to_compute=[0.1, 0.6], iou_thresholds_to_return=[0.1, 0.6], metrics_to_return=[ "AP", ], - convert_annotations_to_type=AnnotationType.RASTER, + convert_annotations_to_type=AnnotationType.BOX, ) - eval_job_raster.wait_for_completion() - - expected = [ - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - ] + eval_job_box.wait_for_completion() - for m in eval_job_raster.metrics: + for m in eval_job_box.metrics: assert m in expected for m in expected: - assert m in eval_job_raster.metrics + assert m in eval_job_box.metrics + # cannot force to polygon as some datasets do not contain this type eval_job_poly = model.evaluate_detection( [dset_box, dset_polygon, dset_raster], iou_thresholds_to_compute=[0.1, 0.6], @@ -3495,90 +3481,17 @@ def test_evaluate_mixed_annotations( convert_annotations_to_type=AnnotationType.POLYGON, ) eval_job_poly.wait_for_completion() + assert eval_job_poly.status == EvaluationStatus.FAILED - expected = [ - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "polygon", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "polygon", "value": "value"}, - }, - ] - - for m in eval_job_poly.metrics: - assert m in expected - for m in expected: - assert m in eval_job_poly.metrics - - eval_job_box = model.evaluate_detection( + # cannot force to raster as some datasets do not contain this type + eval_job_raster = model.evaluate_detection( [dset_box, dset_polygon, dset_raster], iou_thresholds_to_compute=[0.1, 0.6], iou_thresholds_to_return=[0.1, 0.6], metrics_to_return=[ "AP", ], - convert_annotations_to_type=AnnotationType.BOX, + convert_annotations_to_type=AnnotationType.RASTER, ) - eval_job_box.wait_for_completion() - - expected = [ - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "raster", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "box", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "box", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.1}, - "value": 1.0, - "label": {"key": "polygon", "value": "value"}, - }, - { - "type": "AP", - "parameters": {"iou": 0.6}, - "value": 1.0, - "label": {"key": "polygon", "value": "value"}, - }, - ] - - for m in eval_job_box.metrics: - assert m in expected - for m in expected: - assert m in eval_job_box.metrics + eval_job_raster.wait_for_completion() + assert eval_job_raster.status == EvaluationStatus.FAILED