Object Detection Benchmarking

[czaloom]

Changes

• The coco panoptic dataset is now stored publicly in Valor JSONL format.
• Ultralytics YOLO inferences for the coco dataset are now stored publicly in Valor JSONL format.
• Ingestion now supports chunking, this enables the uploading of the entire dataset to the Valor API.
• Key metrics have been expanded upon and broken out in a new results format.
• Bounding Box metrics are now reported.
• Fixed divide by zero edge case in IoU computation.

Example

# BEFORE
"09/08/2024 09:22:54": {
    "number_of_datums": 6,
    "number_of_unique_labels": 28,
    "number_of_annotations": 142,
    "ingest_runtime": "25.1 seconds",
    "eval_runtime": "7.8 seconds",
    "del_runtime": "0.1 seconds"
},

# AFTER
{
    "16/08/2024 16:31:26": [
        {
            "limit": 5000,
            "groundtruths": {
                "dtype": "box",
                "ingestion": 46.67,
                "finalization": 0.97,
                "deletion": 1.84
            },
            "predictions": {
                "dtype": "box",
                "ingestion": 33.86,
                "finalization": 1.07,
                "deletion": 0.62
            },
            "evaluation": {
                "number_of_datums": 4968,
                "number_of_annotations": 63628,
                "number_of_labels": 94,
                "base": 10.67,
                "base+pr": [13](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:14).78,
                "base+pr+detailed": 21.61
            }
        },
        {
            "limit": 5000,
            "groundtruths": {
                "dtype": "box",
                "ingestion": 47.5,
                "finalization": 1.04,
                "deletion": 1.77
            },
            "predictions": {
                "dtype": "box",
                "ingestion": 31.1,
                "finalization": 1.1,
                "deletion": 0.63
            },
            "evaluation": {
                "number_of_datums": 4968,
                "number_of_annotations": 63628,
                "number_of_labels": 94,
                "base": 11.76,
                "base+pr": 13.24,
                "base+pr+detailed": 22.32
            }
        }
    ],
    "16/08/2024 16:36:05": [
        {
            "limit": 5000,
            "groundtruths": {
                "dtype": "polygon",
                "ingestion": 45.45,
                "finalization": 0.86,
                "deletion": 1.78
            },
            "predictions": {
                "dtype": "polygon",
                "ingestion": 30.77,
                "finalization": 0.9,
                "deletion": 0.49
            },
            "evaluation": {
                "number_of_datums": 4915,
                "number_of_annotations": 49765,
                "number_of_labels": 94,
                "base": 15.32,
                "base+pr": 17.05,
                "base+pr+detailed": 24.81
            }
        },
        {
            "limit": 5000,
            "groundtruths": {
                "dtype": "polygon",
                "ingestion": 45.37,
                "finalization": 0.86,
                "deletion": 1.81
            },
            "predictions": {
                "dtype": "polygon",
                "ingestion": 29.65,
                "finalization": 0.9,
                "deletion": 0.54
            },
            "evaluation": {
                "number_of_datums": 4915,
                "number_of_annotations": 49765,
                "number_of_labels": 94,
                "base": [14](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:15).45,
                "base+pr": 16.95,
                "base+pr+detailed": 24.73
            }
        }
    ],
    "16/08/2024 16:37:25": [
        {
            "limit": 6,
            "groundtruths": {
                "dtype": "multipolygon",
                "ingestion": 4.47,
                "finalization": 0.34,
                "deletion": 0.02
            },
            "predictions": {
                "dtype": "multipolygon",
                "ingestion": 2.82,
                "finalization": 0.28,
                "deletion": 0.02
            },
            "evaluation": {
                "number_of_datums": 6,
                "number_of_annotations": 74,
                "number_of_labels": 29,
                "base": 13.98,
                "base+pr": 14.89,
                "base+pr+detailed": -1
            }
        },
        {
            "limit": 6,
            "groundtruths": {
                "dtype": "multipolygon",
                "ingestion": 4.45,
                "finalization": 0.28,
                "deletion": 0.02
            },
            "predictions": {
                "dtype": "multipolygon",
                "ingestion": 2.8,
                "finalization": 5.34,
                "deletion": 0.02
            },
            "evaluation": {
                "number_of_datums": 6,
                "number_of_annotations": 74,
                "number_of_labels": 29,
                "base": 13.9,
                "base+pr": 14.77,
                "base+pr+detailed": -1
            }
        }
    ],
    "16/08/2024 16:38:34": [
        {
            "limit": 6,
            "groundtruths": {
                "dtype": "raster",
                "ingestion": 0.21,
                "finalization": 0.28,
                "deletion": 0.02
            },
            "predictions": {
                "dtype": "raster",
                "ingestion": 0.14,
                "finalization": 0.28,
                "deletion": 0.02
            },
            "evaluation": {
                "number_of_datums": 6,
                "number_of_annotations": 74,
                "number_of_labels": 29,
                "base": [15](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:16).25,
                "base+pr": [16](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:17).33,
                "base+pr+detailed": -1
            }
        },
        {
            "limit": 6,
            "groundtruths": {
                "dtype": "raster",
                "ingestion": 0.[21](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:22),
                "finalization": 0.28,
                "deletion": 0.02
            },
            "predictions": {
                "dtype": "raster",
                "ingestion": 0.14,
                "finalization": 0.[28](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:29),
                "deletion": 0.02
            },
            "evaluation": {
                "number_of_datums": 6,
                "number_of_annotations": 74,
                "number_of_labels": [29](https://github.com/Striveworks/valor/actions/runs/10423021817/job/28868823568?pr=704#step:11:30),
                "base": 15.3,
                "base+pr": 16.38,
                "base+pr+detailed": -1
            }
        }
    ]
}

Fixes

• Rasters are no longer stored as bounding boxes.

[ntlind]

nit: I really like all the extra detail you're adding to the benchmark script. I wish some of these changes were replicated for classification; it's a bit challenging to jump between the classification and object detection outputs now that they're so different.

[czaloom]

nit: I really like all the extra detail you're adding to the benchmark script. I wish some of these changes were replicated for classification; it's a bit challenging to jump between the classification and object detection outputs now that they're so different.

Ive updated the classification benchmark to take the same timeout params and output in a similar format.