From 40f7b3bf12bb3f485e394b2c6786939820fe5aef Mon Sep 17 00:00:00 2001 From: Charles Zaloom <38677807+czaloom@users.noreply.github.com> Date: Fri, 16 Aug 2024 14:40:01 -0400 Subject: [PATCH] Object Detection Benchmarking (#704) --- .../crud/test_create_delete.py | 8 +- api/valor_api/backend/metrics/detection.py | 62 +- api/valor_api/schemas/geometry.py | 24 +- client/unit-tests/schemas/test_geojson.py | 2 +- client/valor/schemas/symbolic/types.py | 6 +- examples/object-detection/coco-yolo.ipynb | 5845 +++++++++++++---- .../integrations/coco_integration.py | 705 +- .../integrations/yolo_integration.py | 464 +- integration_tests/benchmarks/.gitignore | 3 +- .../classification/benchmark_script.py | 284 +- .../object-detection/benchmark_script.py | 599 +- 11 files changed, 6300 insertions(+), 1702 deletions(-) diff --git a/api/tests/functional-tests/crud/test_create_delete.py b/api/tests/functional-tests/crud/test_create_delete.py index 40092e089..4b64ae4f5 100644 --- a/api/tests/functional-tests/crud/test_create_delete.py +++ b/api/tests/functional-tests/crud/test_create_delete.py @@ -1064,15 +1064,11 @@ def test_gt_seg_as_mask_or_polys( assert len(segs.annotations) == 2 assert segs.annotations[0].raster and segs.annotations[1].raster - decoded_mask0 = np.array( - _bytes_to_pil(b64decode(segs.annotations[0].raster.mask)) - ) + decoded_mask0 = segs.annotations[0].raster.array assert decoded_mask0.shape == mask.shape np.testing.assert_equal(decoded_mask0, mask) - decoded_mask1 = np.array( - _bytes_to_pil(b64decode(segs.annotations[1].raster.mask)) - ) + decoded_mask1 = segs.annotations[1].raster.array assert decoded_mask1.shape == mask.shape np.testing.assert_equal(decoded_mask1, mask) diff --git a/api/valor_api/backend/metrics/detection.py b/api/valor_api/backend/metrics/detection.py index f20df6ed7..4d1b62b87 100644 --- a/api/valor_api/backend/metrics/detection.py +++ b/api/valor_api/backend/metrics/detection.py @@ -7,7 +7,7 @@ from typing import Sequence, Tuple from geoalchemy2 import functions as gfunc -from sqlalchemy import CTE, and_, func, or_, select +from sqlalchemy import CTE, and_, case, func, or_, select from sqlalchemy.orm import Session, aliased from valor_api import enums, schemas @@ -458,10 +458,14 @@ def _compute_detailed_curves( if misclassification_detected: fn["misclassifications"].append( (dataset_name, datum_uid, gt_geojson) + if gt_geojson is not None + else (dataset_name, datum_uid) ) else: fn["no_predictions"].append( (dataset_name, datum_uid, gt_geojson) + if gt_geojson is not None + else (dataset_name, datum_uid) ) if label_id in predictions_per_label: @@ -501,10 +505,14 @@ def _compute_detailed_curves( if misclassification_detected: fp["misclassifications"].append( (dataset_name, datum_uid, pd_geojson) + if pd_geojson is not None + else (dataset_name, datum_uid) ) elif hallucination_detected: fp["hallucinations"].append( (dataset_name, datum_uid, pd_geojson) + if pd_geojson is not None + else (dataset_name, datum_uid) ) # calculate metrics @@ -1158,14 +1166,22 @@ def _annotation_type_to_column( select( gt_pd_counts.c.gt_annotation_id, gt_pd_counts.c.pd_annotation_id, - func.coalesce( - gt_pd_counts.c.intersection - / ( + case( + ( gt_pd_counts.c.gt_count + gt_pd_counts.c.pd_count - gt_pd_counts.c.intersection + == 0, + 0, + ), + else_=( + gt_pd_counts.c.intersection + / ( + gt_pd_counts.c.gt_count + + gt_pd_counts.c.pd_count + - gt_pd_counts.c.intersection + ) ), - 0, ).label("iou"), ) .select_from(gt_pd_counts) @@ -1183,7 +1199,10 @@ def _annotation_type_to_column( select( gt_pd_pairs.c.gt_annotation_id, gt_pd_pairs.c.pd_annotation_id, - iou_computation.label("iou"), + case( + (gfunc.ST_Area(gunion) == 0, 0), + else_=iou_computation, + ).label("iou"), ) .select_from(gt_pd_pairs) .join( @@ -1209,10 +1228,7 @@ def _annotation_type_to_column( gt.c.label_id.label("gt_label_id"), pd.c.label_id.label("pd_label_id"), pd.c.score.label("score"), - func.coalesce( - gt_pd_ious.c.iou, - 0, - ).label("iou"), + gt_pd_ious.c.iou, gt.c.geojson.label("gt_geojson"), ) .select_from(pd) @@ -1576,14 +1592,22 @@ def _annotation_type_to_column( select( gt_pd_counts.c.gt_annotation_id, gt_pd_counts.c.pd_annotation_id, - func.coalesce( - gt_pd_counts.c.intersection - / ( + case( + ( gt_pd_counts.c.gt_count + gt_pd_counts.c.pd_count - gt_pd_counts.c.intersection + == 0, + 0, + ), + else_=( + gt_pd_counts.c.intersection + / ( + gt_pd_counts.c.gt_count + + gt_pd_counts.c.pd_count + - gt_pd_counts.c.intersection + ) ), - 0, ).label("iou"), ) .select_from(gt_pd_counts) @@ -1601,7 +1625,10 @@ def _annotation_type_to_column( select( gt_pd_pairs.c.gt_annotation_id, gt_pd_pairs.c.pd_annotation_id, - iou_computation.label("iou"), + case( + (gfunc.ST_Area(gunion) == 0, 0), + else_=iou_computation, + ).label("iou"), ) .select_from(gt_pd_pairs) .join( @@ -1627,10 +1654,7 @@ def _annotation_type_to_column( gt.c.label_id.label("gt_label_id"), pd.c.label_id.label("pd_label_id"), pd.c.score.label("score"), - func.coalesce( - gt_pd_ious.c.iou, - 0, - ).label("iou"), + gt_pd_ious.c.iou, gt.c.geojson.label("gt_geojson"), (gt.c.label_id == pd.c.label_id).label("is_match"), ) diff --git a/api/valor_api/schemas/geometry.py b/api/valor_api/schemas/geometry.py index 409560f98..9cacc82d3 100644 --- a/api/valor_api/schemas/geometry.py +++ b/api/valor_api/schemas/geometry.py @@ -11,7 +11,6 @@ ST_GeomFromText, ST_MakeEmptyRaster, ST_MapAlgebra, - ST_SnapToGrid, ) from pydantic import ( BaseModel, @@ -1028,8 +1027,8 @@ def to_psql(self) -> ScalarSelect | bytes: if self.geometry: empty_raster = ST_AddBand( ST_MakeEmptyRaster( - self.width, - self.height, + self.width, # width + self.height, # height 0, # upperleftx 0, # upperlefty 1, # scalex @@ -1038,23 +1037,18 @@ def to_psql(self) -> ScalarSelect | bytes: 0, # skewy 0, # srid ), - "1BB", - ) - geom_raster = ST_AsRaster( - ST_SnapToGrid( - ST_GeomFromText(self.geometry.to_wkt()), - 1.0, - ), - 1.0, # scalex - 1.0, # scaley "1BB", # pixeltype - 1, # value - 0, # nodataval ) return select( ST_MapAlgebra( empty_raster, - geom_raster, + ST_AsRaster( + ST_GeomFromText(self.geometry.to_wkt()), + empty_raster, + "1BB", + 1, + 0, + ), "[rast2]", "1BB", "UNION", diff --git a/client/unit-tests/schemas/test_geojson.py b/client/unit-tests/schemas/test_geojson.py index 7141630a7..2e326aa74 100644 --- a/client/unit-tests/schemas/test_geojson.py +++ b/client/unit-tests/schemas/test_geojson.py @@ -105,7 +105,7 @@ def test_multipolygon(): MultiPolygon([coords]) # type: ignore - testing with pytest.raises(TypeError): MultiPolygon([[coords], 123]) # type: ignore - testing - with pytest.raises(TypeError): + with pytest.raises(ValueError): MultiPolygon([[[coords]]]) # type: ignore - testing diff --git a/client/valor/schemas/symbolic/types.py b/client/valor/schemas/symbolic/types.py index 9222a2f9d..80a81a019 100644 --- a/client/valor/schemas/symbolic/types.py +++ b/client/valor/schemas/symbolic/types.py @@ -769,12 +769,14 @@ def __init__( @classmethod def __validate__(cls, value: typing.Any): - if not isinstance(value, tuple): + if not isinstance(value, (tuple, list)): raise TypeError( f"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'" ) elif len(value) != 2: - raise ValueError("") + raise ValueError( + "A point should contain only two x-y coordinates." + ) for item in value: if not isinstance(item, (int, float, np.floating)): raise TypeError( diff --git a/examples/object-detection/coco-yolo.ipynb b/examples/object-detection/coco-yolo.ipynb index 23efbcc8a..5fa881112 100644 --- a/examples/object-detection/coco-yolo.ipynb +++ b/examples/object-detection/coco-yolo.ipynb @@ -37,8 +37,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/czaloom/valor/.env-velour/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" + "WARNING:root:The Valor client version (0.31.1.dev6+gd4c07731.d20240731) is older than the Valor API version 0.31.4.dev3+g0a6b90ae.d20240813\t==========================================================================================\n", + "\t== Running with a mismatched client != API version may have unexpected results.\n", + "\t== Please update your client to \u001b[1;0.31.4.dev3+g0a6b90ae.d20240813\u001b[0;31m to avoid aberrant behavior.\n", + "\t==========================================================================================\n", + "\u001b[0m\n" ] }, { @@ -56,7 +59,7 @@ "from pathlib import Path\n", "import pandas as pd\n", "\n", - "from valor import Client, Model, Annotation, Label, Filter, connect\n", + "from valor import Client, Dataset, Model, Annotation, Label, Filter, connect\n", "from valor.enums import TaskType, AnnotationType\n", "from valor.schemas import And, Eq\n", "from valor.viz import create_combined_segmentation_mask\n", @@ -105,7 +108,7 @@ "id": "94798123", "metadata": {}, "source": [ - "This block utilizes `create_dataset_from_coco_panoptic` from `integrations/coco_integration.py` to download, extract, and upload the COCO Panoptic validation dataset to Valor." + "This block utilizes `get_instance_groundtruths` from `integrations/coco_integration.py` to download, extract, and upload the COCO Panoptic validation dataset to Valor." ] }, { @@ -118,25 +121,76 @@ "name": "stdout", "output_type": "stream", "text": [ - "coco already exists at /home/czaloom/valor/examples/coco!\n" + "gt_objdet_coco_bbox.jsonl already exists locally.\n" ] }, { - "name": "stderr", + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create the dataset in Valor\n", + "valor_dataset_bbox = Dataset.create(\"coco-box\")\n", + "\n", + "# retrieve chunks containing valor.Groundtruth objects and upload them.\n", + "for chunk in coco.get_instance_groundtruths(\n", + " dtype=AnnotationType.BOX,\n", + " chunk_size=100,\n", + " limit=100,\n", + " from_cache=True,\n", + "):\n", + " valor_dataset_bbox.add_groundtruths(chunk)\n", + "\n", + "# finalize the data\n", + "valor_dataset_bbox.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b525b611", + "metadata": {}, + "outputs": [ + { + "name": "stdout", "output_type": "stream", "text": [ - "Formatting: 100%|██████████| 5/5 [00:00<00:00, 120.60it/s]\n", - "Uploading: 100%|██████████| 5/5 [00:00<00:00, 10.79it/s]\n" + "gt_objdet_coco_raster_bitmask.jsonl already exists locally.\n" ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "# create the dataset in Valor\n", - "valor_dataset = coco.create_dataset_from_coco_panoptic(\n", - " destination=Path(\"./\").absolute().parent / Path(\"coco\"),\n", - " limit=5,\n", - " delete_if_exists=True\n", - ")" + "valor_dataset_raster = Dataset.create(\"coco-raster\")\n", + "\n", + "# retrieve chunks containing valor.Groundtruth objects and upload them.\n", + "for chunk in coco.get_instance_groundtruths(\n", + " dtype=AnnotationType.RASTER,\n", + " chunk_size=100,\n", + " limit=100,\n", + " from_cache=True,\n", + "):\n", + " valor_dataset_raster.add_groundtruths(chunk)\n", + "\n", + "# finalize the data\n", + "valor_dataset_raster.finalize()" ] }, { @@ -147,53 +201,50 @@ "source": [ "## Defining Our Model\n", "\n", - "With our `Dataset` in Valor, we're ready to create our `Model` object and add `Predictions` to it." + "With our `Dataset` in Valor, we're ready to create our `Model` object and add `Predictions` to it. This block utilizes `get_instance_predictions` from `integrations/yolo_integration.py` to run inferences over the COCO Panoptic validation dataset. To save on time, the default behavior of this function is to draw from a cache of precomputed inferences." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "0e2750a6", "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "100%|██████████| 5/5 [00:03<00:00, 1.30it/s]\n" + "pd_objdet_yolo_bbox.jsonl already exists locally.\n", + "pd_objdet_yolo_raster.jsonl already exists locally.\n" ] } ], "source": [ - "# define the model in Valor. note that we can use any name we'd like\n", - "valor_model_bbox = Model.create(\"yolov8n-box\")\n", - "valor_model_seg = Model.create(\"yolov8n-seg\")\n", - "\n", - "inference_engine = ultralytics.YOLO(f\"yolov8n-seg.pt\")\n", - "\n", - "for datum in tqdm(valor_dataset.get_datums()):\n", - "\n", - " image = coco.download_image(datum)\n", + "# define the model in Valor. note that we can use any name we'd like.\n", + "valor_model = Model.create(\"yolov8n\")\n", "\n", - " results = inference_engine(image, verbose=False)\n", + "# retrieve chunks containing bounding box predictions and upload them.\n", + "for chunk in yolo.get_instance_predictions(\n", + " dtype=AnnotationType.BOX,\n", + " coco_uids=[datum.uid for datum in valor_dataset_bbox.get_datums()],\n", + " chunk_size=200,\n", + " from_cache=True,\n", + "):\n", + " valor_model.add_predictions(valor_dataset_bbox, chunk)\n", "\n", - " # convert result into Valor Bounding Box prediction\n", - " bbox_prediction = yolo.parse_detection_into_bounding_box(\n", - " results, # raw inference\n", - " datum=datum, # valor datum\n", - " label_key='name', # label_key override\n", - " )\n", - "\n", - " # convert result into Valor Raster prediction\n", - " raster_prediction = yolo.parse_detection_into_raster(\n", - " results, # raw inference\n", - " datum=datum, # valor datum\n", - " label_key='name', # label_key override\n", - " )\n", + "# retrieve chunks containing bitmask predictions and upload them.\n", + "for chunk in yolo.get_instance_predictions(\n", + " dtype=AnnotationType.RASTER,\n", + " coco_uids=[datum.uid for datum in valor_dataset_raster.get_datums()],\n", + " chunk_size=5,\n", + " limit=5,\n", + " from_cache=True,\n", + "):\n", + " valor_model.add_predictions(valor_dataset_raster, chunk)\n", "\n", - " # add predictions to the model\n", - " valor_model_bbox.add_prediction(valor_dataset, bbox_prediction)\n", - " valor_model_seg.add_prediction(valor_dataset, raster_prediction)" + "# finalize the inferences for a dataset\n", + "valor_model.finalize_inferences(valor_dataset_bbox)\n", + "valor_model.finalize_inferences(valor_dataset_raster)" ] }, { @@ -209,7 +260,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "14939a3a", "metadata": {}, "outputs": [ @@ -221,13 +272,13 @@ "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "groundtruth_139 = valor_dataset.get_groundtruth('139')\n", + "groundtruth_139 = valor_dataset_raster.get_groundtruth('139')\n", "assert groundtruth_139\n", "coco.download_image(groundtruth_139.datum)" ] @@ -243,19 +294,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "737e3e25-aa4a-4934-ad5f-da770bffa44a", "metadata": {}, "outputs": [ { "data": { - "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAGqAoADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5/ooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiux0bQLRLaK5nUTvIiuocfKoI6Y79ep/SsfxDpP9n3Pnxf6iZiQAuNh9PTHp/wDWrrqYKrTpe1Z0zwtSFP2jMaiiiuQ5gooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPUaxvFE3laKybc+a6pnPT+L/2WqujeI4TBHb30jLMOPNfkNzxk9j9fTOaz/EOtRagqW1sN0KMHMhBGTjsPTn/AD3+hxGMpSw7cXq1t1ParYmnKg2nqzBooor548UKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACrWn2Euo3LwQsissEs5LkgbY42kbp3wpx74qrXV+BLCK8udenkZw1noV7PGFIwWMfl8+2JD+OKxxFT2VKU+xUVd2OUooorYkKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKciNI6oilnY4VVGST6CgBtFakHhzV7hC6WMgAOP3hCH8mINULi3mtLh4J4zHKhwyntVypziryTSM41ac3yxkm/UioooqDQKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoor6w+H1hFp3w/wBCghZ2VrRJyXIJ3SfvGHHbLnHtivPzHHrBwUuW933t/mB8n0V9q1Vv9NsNVgWDUbK2vIVbeI7iJZFDYIzhgecE8+9eSuI1fWn+P/AFc+NKK+uv+EO8L/8AQt6P/wCAMX/xNH/CHeF/+hb0f/wBi/8Aia0/1ipfyMLnyLRX11/wh3hf/oW9H/8AAGL/AOJrLu/hd4Lvbp7iXQoVd8ZEMskScDHCowUdOwqo8Q0L+9B/h/wAufLNFfUH/CpPA/8A0BP/ACbn/wDi6P8AhUngf/oCf+Tc/wD8XV/6wYb+WX3L/MLny/RX0zd/B3wXc2rwxadNaO2MTQ3Mhdec8byy89OQetZf/CiPC/8Az/6x/wB/ov8A43Vxz7CNa3Xy/wCCFz56or6F/wCFEeF/+f8A1j/v9F/8bo/4UR4X/wCf/WP+/wBF/wDG6r+3cJ3f3Bc+eqK9xu/gDavdO1l4hmhtzjZHNaiRxxzlgyg857D+tQf8M/f9TP8A+SH/ANsrRZzgmvj/AAf+Q7nitFe1f8M/f9TP/wCSH/2yobv4A3SWrtZeIYZrgY2RzWpjQ885YMxHGex/rTWcYJu3P+D/AMgueN0V6h/wojxR/wA/+j/9/pf/AI3XL+MPAmqeCfsX9pT2cv2vf5f2Z2bGzbnO5R/eFdFLHYarNQhNNsDl6KKK6wCiiigAooooAKKKKACiiigArtfCX+i+BfG2pQ/LdpbW1qsnXEU0u2RcHjkAc4yO2K4qvRdAsIo/gf4s1AM/nT3dvAykjaFjeNhj3zI2foK4sfJKnFPrKC/8mX6GlJav0f5HnVFFFdpmFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRW7pvhS/1C3juC0cELnjzM7iv94DH5ZIz+tWtQ8GT2drJcRXkcqRozuHQocAZ4657+ldKwldx51HQ5HjsOp8jnqcxRRRXMdYV3HgzTrT7IdQz5lzuKcj/Vew9yCOffHrnh66jwVfmLUJLJ3/dzKWRTk/OPTsOM5+grswEoqvHmX/DnBmcZyw0uR7fl1O7rF8R6IurWZeKMfbIx+7bONw7qf1x7+nNbVFfR1KcakXCWzPk6VWVKanDdHjlFXNX/AOQ1f/8AXxJ/6Eap18lJcsmj7mEuaKl3CiiipKCiirml6Xea1qMWn6fD511LnZHuC5wCx5JA6A0JX0QpSUU23ZIp0V0d94B8V6d5fn6Hdv5mcfZwJ8Yx12E469+v4VhXVpc2Ny9td28tvOmN0UyFGXIyMg8jgg03FrdEQrU6nwST9GQ0UUUjQKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvrrwd/wAiPoH/AGDbf/0WtfItfXXg7/kR9A/7Btv/AOi1r53iL+FD1Ezbooor5QQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFeKftA/8y7/28/8AtKva68U/aB/5l3/t5/8AaVenk3++w+f5MaPFaKKK+6GFFFFABRRRQAUUUUAFFFdR4E8H/wDCa65Npv277H5Vs0/meT5mcMq4xuH97rntWdatCjB1KjsluOMXJ2Q/wd4ai8WWWtafbxO2tRQJdWTbwqFVbbIjZ7sHXHuvJAznpND/AOTevEv/AGEk/wDQreu78F/Cufwb4hTVIvEHnoY2img+xBPMU843bzjDBTwO2O9aPj7wS3iTRrTTNKhtrMTaoLq7mVAoA8tw8hAxvYkqPUkjJAyR8riM3o1cSoKV4c0ZX10tute9unfqdsaElC9tdUfMlFek/E/StL8IWum+F9ISbD5vruaYKzzkkpFlgAflHmDAAHzZ5JJrzavpsLiFiKSqxVk9vQ45x5XysKKKK6CQooooAKKKKACiiigAooooAKKKKACiiigD03w3epe6FbFBhoVELj0KgD9Rg/jV3UY/O0y7i3om+F13OcKuVPJPYVn+F7H7DoUOWy0/7488DcBj9APxzWd401G5treG0hykc4bzHH8QGPlHOe/PHp719N7X2eFU6nY+PVFVca4Un1OGooor5k+wCtTw5K0XiGzZIzIS+3aOwIIJ/AHP4Vl1s+Ff+RltP+B/+gNW2H/jQ9V+ZhitKE79n+R6VRRRX1p8OcP4103yrmPUUHyy/JJ/vAcHr3A/8d965SvWNTsxqGmXFqQCZEIXcSAG6qTj3xXlLo0bsjqVdThlYYIPoa+dzKh7OrzraX5n1WUYj2tHke8fy6f5DaKKK849UK6/4Xf8lG0r/tt/6JeuQrr/AIXf8lG0r/tt/wCiXq6fxr1ObG/7tU/wv8j6OqnqelWGs2bWmo2kVzA2fkkXODgjIPUHBPIwRmrlFes1c/PoycXdOzPlHW7KPTdf1GwhZ2itrqWFC5BYqrEDOO/FUK2PFn/I465/2EJ//RjVj148t2fo1Jt04t9kFFFFI0CiiigAooooAKKKKACiiigAooooAKKKKACvrrwd/wAiPoH/AGDbf/0WtfItfX3hSGW28HaJBPG8U0enwI8bqVZWEaggg9CD2r53iL+HD1Yma9FFFfKCCiiigAooooAKKKKACiiigAooooAKKKKACiiigArxT9oH/mXf+3n/ANpV7XXgPx5u538VabZM+beKy81EwOGd2DHPXkIv5fWvVyWLeNi+1/yGjyiiiivuBhRRRQAUUUUAFFFFABWv4Z8Q3nhfXrbVLN3BjYCWNW2iaPI3IeDwQPQ4OCOQKyKKicI1IuE1dMabTuj7SyKN3NUNI1D+1tEsNS8ryvtdtHP5e7ds3qGxnAzjPXFZHj+/i07wBrk8yuytaPAAgBO6T92vXtlhn2zXzOEyLDezUpXba69L+S7erPYcrLmPL/j3p/l65pGpebn7RbPB5e37vltuznPOfN6Y42+/Hkddx4s8axeMvCemjUd669p85jJQAR3ETr80mAOGBRARwOSR1wvD17OV0qlHCxpVfijdfjpb5Hl1mpTcl1CiiivQMgooooAKKKKACiiigAor3L4OeFtKvvCdzf6po1tczTXbLHJdQBwY1VcbdwIxuLgkdSMHpxzHxxmil8c26RyI7RWEaSKrAlG3u2D6HDA/Qj1qebWxs6LVPnbPNKKKKoxCiiigD1jS0aPSLJHUq6wIGVhgg7RwaqeJLJL3QrkOcNCpmQ+hUE/qMj8agsPEemJZ2kFxfD7R5UYcsGPzFRnLYx9efrU+s6jZLoVy32qErNC6xFXB3nGOMdeT+FfUOdKdBq6asfGxp1oYhS5Wnft5nmVFFFfLn2QVu+EIGl8QxOpAEKM7Z7jG3j8WFYVd34O0g21s2oSjEk64jXBBVM9fxwD9APWuvA0nUrxt01OHMaypYeV+un3nUUUUV9QfHBXnPi61W2152TAE6CXaFxg8g/XJGfxr0auE8cRONTt5iv7todqnPUhiT/6EPzrzszjehfsz1MnlbE27pnL0UUV86fVhXX/C7/ko2lf9tv8A0S9chXX/AAu/5KNpX/bb/wBEvV0/jXqc2N/3ap/hf5H0dRRRXrn54fLXiz/kcdc/7CE//oxqx67HxD4T1y98T6xcwWDNFJfTshaRFyPMbnBINYmoeGdY0u1NzeWTRwggFw6tjPTOCcfWvLnQqq8uV29D9Oo4auqEZuDtZa2fYyaKKKxJCiiigAooooAKKKKACiiigAooooAKKKKACvtWviqvtWvmOI/+XXz/AEEwooor5kQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFfPXx3/5Hiy/7Bqf+jJa+ha+evjv/wAjxZf9g1P/AEZLXsZF/va9GNHl9FFFfajCiiigAooooAKKKKACiiigD1Xw/wDGy60fRLXTbrRIbr7LEkMUkVwYvkVQo3Aq2W45IIHPQVB4v+Ln/CVeFbrSF0j7G88ke5jN5oKKdxwcLhtyp2IILdOK8xorOFKEFaKsa+3qW5b6BRRRWhkFFFFABRRXtd58M/Dlzs8qK5tNuc+TMTu+u/d09sda5sRi6eHa5+p1YbCVMQnydDxSivV7z4S2L7PsWp3MOM7/ADkWXPpjG3Hf1qK3+EsEc6m51OSeE5DCJBEy8cEE7s88Y465zxg4/wBpYa1+b8GbPLMTe3L+KPLaK9N/4VD/ANRz/wAlP/s6P+FQ/wDUc/8AJT/7Oq/tHDfzfg/8if7NxX8v4r/M9P8AAklr4a+Gmh/2tf2dqkkfmLJLMEQ+azSKuWx821uR7Hr1rxD4nXlrf/EPVLqzuYbm3fytksLh0bESA4I4PII/Cs3xT4dk8MasLF7hbhXiWVJAu0kEkcjnByD3Pb6ViV003GS54u6ZlXnJL2Ulax9GQ+FdHuPhPHPBoFjLqMmhh0kSzRpWlMGQQQMli345r5zr7A0Gwl0vw7pmnzsjTWtpFA7ISVLKgU4zjjIr5M1jT/7J1u/03zfN+yXMkHmbdu/YxXOMnGcdM0Qe5WJhZRZSooorQ5Aoq1p1hPqmo29jbLumncIvBIHqTgHgDknsAa9QtPhNpqREXmo3c0m7hoVWMY9MENz15zXNXxVKhZTep04fCVa93BaI8lqxZ2VzqFwILWIySYJwOMD1JPAr1uP4V6DHKjtPfyKrAlGlXDD0OFBx9CKfqnw/sQluuhLJp0zy4muY5nYrHtY4wX5ywQcVjDNMNzpSvY3nlWK5G42v0OJtPB0NqDcavdxrEh5VG2r1GMscdeRj9a0bfxJ/amtw2VjG/wBn+ZpZivOB0x6A4Ayefm7GpfEXgF7DRWuTdanq+pGRY4vKj3Kq9TuX5mxw3IPUjiuX0q7l8L37f2hplwksqADzAY2VM8kKRznH6da9vCZnQnJKj7sb6vq7Hz+NynEQi5Yj3p20XRX/AA8z0WisXTvFFhqd4trEk6SMCV8xRg45xwT2z+VbVfR06kKi5oO6PlKtGdKXLUVmFcj47icw2UwX92rOrHPQkAj/ANBP5V11c740ieTQlZFyscys5z0GCP5kVhjY82Hkjpy6XLioPz/PQ8+ooor5c+yCuv8Ahd/yUbSv+23/AKJeuQrr/hd/yUbSv+23/ol6un8a9Tmxv+7VP8L/ACPo6iiivXPzw4m+/wCQhc/9dW/max9d/wCRe1P/AK9Jf/QDVe/8X6Nbahc293fbbmKVkmXyXOHBIYcLjrnpWXq3i/QrnRr63hvt0stvIiL5LjJKkAcrXfOvR9m1zrbuj+gIYnDwwqh7SLajbRrseWUUUV8sfFhRRRQB7p8F9A0bVfB13PqOk2F5MuoOgkuLZJGC+XGcZYHjJPHvXo3/AAh3hf8A6FvR/wDwBi/+Jrm/g1YRWfw5tZ42cteTyzyBiMBg3l4HtiMevOa7+vg8wrz+tVOWTtd9RGJ/wh3hf/oW9H/8AYv/AImj/hDvC/8A0Lej/wDgDF/8TW3RXF7er/M/vEYn/CHeF/8AoW9H/wDAGL/4mj/hDvC//Qt6P/4Axf8AxNbdFHt6v8z+8DE/4Q7wv/0Lej/+AMX/AMTR/wAId4X/AOhb0f8A8AYv/ia26KPb1f5n94GJ/wAId4X/AOhb0f8A8AYv/ia5H4oeGtB0/wCHWq3Vlomm21wnk7JYbVEdcyoDggZHBI/GvSa4r4t/8kw1j/tj/wCjo66sFWqPE005P4l18xny/X2rXxVX2rXscR/8uvn+gMKKKK+ZEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXz18d/wDkeLL/ALBqf+jJa+ha+ZvjFdz3PxJv4pn3JbRwxQjAG1TGHxx1+Z2PPrXtZDFvF37J/oNHB0UUV9mMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvpuvmSvpHTbz+0NLtL3y/L+0QpLsznbuUHGe/WvFzhO0H6/oe5krV5r0/UtVV1FYjYyPNBJOsOJxFECXZkIdQoHU5UYHerVFeInZ3PcaurFNtVsFtzOLyF4wm/MbhyRjPAGSfw61Q/4S3R/wDnpdf+AM//AMRXH6ZbNZWRs3lMxtppYPMIxuCSMo45xwOlXK9qOW0urbOulhXUpxmna6T+/wCZzXxBkbXNegudOt7qaFbVYy32aRcMGc4wyjsRXN6N5Fj4m01tWi2WsV3E1yk0RI8sMC2VxyNueMc16TRXpUrU4KC2R59fh2NWbqKpZvy/4J61/wAJh4Y/6GPSP/A6L/4qvm3x9eWeoeO9XurD7MbZ5/la2OUcgAFgcDJJBJI4yTgsOT3FFVGVgq8PuorOr/5L/wAE8jor1yir9p5GH+rH/T3/AMl/4Jj/AAs0Pz76fWpR8ltmKHnrIR8x69lOORg7/avV6r2MH2axhi27SqDcM556n9c1Yr5fF13WquXTobYfDRw8PZxd/PuFU7m9jtb+1imurSGOdWVUlk2ySSZXaEB6jls98lauVmahGJdV01WJADO3HqAD/SsqUVKVn2f5HQoczte3/A1NOiiisxFHWP8AkFTf8B/9CFcYNRsjcrbLdQtOzFfLVwWBAJOQOnQ9a7m//wCQfc/9cm/ka4lfvD619bw9mDoUXRUb3le/qkv0Pn844aWZSli3U5eSNrWvtd738+xNWL4sdV8OXIZgCxQKCep3A4H4A/lWpd+f9in+zf6/y28vp97HHXjr615jfa1qGpQrDd3HmRq24DYo5wR2Hua+sx+JjSg4NayTPznLcJKtUVRNWi1fuUKKKK+bPrQru/hFY/a/HkU/mbPsdvJPjbnfkeXj2/1mfw964SvR/gr/AMjjef8AYPf/ANGR1pRX7xHFmMmsJUa7Hu1FFFesfAHy14s/5HHXP+whP/6MasetjxZ/yOOuf9hCf/0Y1Y9ePL4mfo9D+FH0QUUUVJqFFFFAH1H8KYZYPhnoyTRvGxWVwrqQSrSuynnsQQQe4IrsqxPB3/Ij6B/2Dbf/ANFrW3X5zipc1ecu7f5khRRRWABRRRQAUUUUAFcV8W/+SYax/wBsf/R0ddrXFfFv/kmGsf8AbH/0dHXTgv8Aeaf+JfmB84aBYRar4j0vTp2dYbq7igdkIDBWcKSM55wa+xK+RfB3/I8aB/2Erf8A9GLX11Xs8RN+0gvJjYUUUV86IKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvl/4t/wDJT9Y/7Y/+iY6+oK+X/i3/AMlP1j/tj/6Jjr3eH/8AeZf4X+aGjiqKKK+wGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABX0R4b/wCRW0j/AK8of/QBXzvX0jptn/Z+l2ll5nmfZ4Ui34xu2qBnHbpXj5u1yxR7WTJ8836FqiiivCPfPJtU1e28PeIdWsbk3Exa6adSijCrIA+3luxY/wA+9OTxTpDIrG6KkjJUxtke3AxTPilpNyNZi1KKzb7K1uqyzonG8MR8xHQ4KAE9eg6V59X1WFaqUYy8jyJ5zisJN0Uk0trp7fej0X/hJ9H/AOfz/wAhP/hU1tr2mXcwhhu0LnoGBXPbAyBk+1eaVb0r/kMWP/XxH/6EK3dNF0eIsTKpGMoxs35/5nqNZ+oa1Y6Y/l3MpEmzeECElhz+Hbua0o0aWRY0GWYhQPUmuc8QeDPE2oavJNFp/mQhVWM+dGOMZP8AFnqTWLqU4u05JfM+hzPF1cNR5qKvJvtf56Df+Ez07/njdf8AfK//ABVH/CZ6d/zxuv8Avlf/AIqs9/h14nVUI09WLLkgTx5U5Iwct14zxnqO+QMm58Oa3aecZ9JvUWHd5j+QxQAdTuAxj3ziqjUoy+GSfzPmpZzmUfiVv+3T6Iooor5I9gKzJQH8RQh3I2QFkXPBYkg/p/KtOuH1e8eD4saLE9y0du9qQUMmFZj5oHHQknAHviujDR5nK3Z/kTOuqKu+un3ncUUUVzlDJVjaF1lx5ZUhsnAx35rhF+8PrXX61dx2WkTyyBirbYhtHOXYIPwywrkF5YV7uSwblp1aRdSpGngq85bKLf4MmryfVLcWmq3cCxmNElYIpzwueOvtivWK8+8Z2vk62JgH2zxhiT03DggfgB+dfb5rC9JS7M/HMlqctZw7r8jnaKKK8A+nCvR/gr/yON5/2D3/APRkdecV6P8ABX/kcbz/ALB7/wDoyOtaP8RHDmf+6VPQ92ooor1T4E+WvFn/ACOOuf8AYQn/APRjVj1seLP+Rx1z/sIT/wDoxqx68eXxM/R6H8KPogoooqTUKKKns7SfUL63srVPMuLiRYokyBuZjgDJ4HJ70m0ldgfY9naQafY29lap5dvbxrFEmSdqqMAZPJ4Hep6KK/M223dkhRRRQAUUUUAFFFFABXFfFv8A5JhrH/bH/wBHR12tcV8W/wDkmGsf9sf/AEdHXTgv95p/4l+YHzhoF/FpXiPS9RnV2htbuKd1QAsVVwxAzjnAr3T/AIXv4X/58NY/78xf/HK+eqK+2xeX0cU1Kr0KPoX/AIXv4X/58NY/78xf/HKP+F7+F/8Anw1j/vzF/wDHK+eqK5f7CwnZ/eKx9Jaf8afB975nnzXlhsxt+025O/Oenl7umO+Over3/C2/A/8A0G//ACUn/wDiK+X6KzlkGFbunJfNf5BY+oP+Ft+B/wDoN/8AkpP/APEUf8Lb8D/9Bv8A8lJ//iK+X6KX+r+G/ml96/yCx9Qf8Lb8D/8AQb/8lJ//AIiprT4o+C726S3i12FXfODNFJEnAzyzqFHTua+WaKT4ew1tJS/D/ILH1bqHxG8H6Z5fn+ILN/Mzt+zMZ8Yx18sNjr3xn8Kzpvi94JigkkTVnmZVLCNLWUM5A6DcoGT05IHvXzHRRHh/Dpe9J/h/kFj6F/4Xv4X/AOfDWP8AvzF/8crOv/j7YRzqNO0G5uIduS1xOsLBsnjCh+MY5z+FeF0VvHJMGndxb+bCx7Jd/H66e1dbLw9DDcHGySa6MiDnnKhVJ4z3H9Ky/wDhe/ij/nw0f/vzL/8AHK8voreOVYOKsqa/FjO8u/jF40ubp5otRhtEbGIYbaMovGON4ZuevJPWs6/+JfjLUYFhn1+5RVbcDbhYGzgj70YUkc9M4/KuUoreOCw0bWpr7kBt/wDCY+KP+hk1j/wOl/8Aiqx5ppbmeSeeR5ZpGLvI7FmZickknqSe9MoraNOEfhVgCiiirAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvorQ7281DSYLi+tVt7hlGRHKsiOMZDIVJ+U/5z1PzrXf3PijV/Dfhvw1b29ztuDC880UqiQPEX/dAk9BtBGAQQPTivOzCg6yjGO9/wBP+AelluIVBzlK9rfr/wAH8z1yivHv+Fr67/z6ad/37f8A+LrT/wCFvf8AUD/8m/8A7CvKlluJWyv80evHNMM93b5M9OorkdD+Imka1eQWRSa1uZVGPO27C/8AdDA8n0yBn6kCm+IfiHY6DfXOn/Y7me8h28ZVYzkBvvZJ6H+71/OsPqlbn5OXU3+t0OT2nNob8mgaNNK8sukWEkjsWZ2tkJYnqSccmoLjwvo0kDLFpVjFKMNG8cKoVYHIOVGeoFeaan8Udau2IsY4bGPcCCFEj4xyCWGCM8/dHb8b3gvxb4i1nxTa2l1eNPalXaZRAgAAU4JIXIG7b+YHeuv6liqcPaOVra7s5IY/CyqKEY3bfZG9Yf8AIQtv+uq/zFdvXCRzNHcLOAu5XDgYwM5z0FNg+KFgl/NaajZTQeXL5QmiPmKcEgsRwQO+BuP9dsfh6tVqUFex7uZVYUnF1Ha53tFZ2ma9pWsqDp9/DOxUt5YbDgA4JKn5gM+o7j1p2razYaHarc6jP5MLOIw2xmyxBOMKD2BryPZz5uW2vY4/aQ5ee6t3L9Fcx/wsLwt/0FP/ACXl/wDiaP8AhYXhb/oKf+S8v/xNafVq/wDI/uZn9aofzr70dPXm3xF8jTfFPh7WJPMbY48xVwfkjdW4Hr87d/TpXWad4y0DVb6Oysr/AM24kzsTyZFzgEnkqB0Brjvi9/zBv+2//tOunBU5RxMYzTV7/kzlx1SEsNKcGna35o9MkkjhieWV1jjRSzOxwFA6knsKdXH+J9Ylb4Zm+aSOG4vbWEYGMMZNu9VBz/CW98DPbNePWepX2n7/ALFe3Ntvxv8AJlZN2OmcHnqfzqsNl8q0HK9rOwsTmMaE1HlvdXPavHV3JbaRZRIFK3OoQRPkcgBt/HvlB+tZKffFT215fa18Nba7uWa4upJkZ2VACQtyOcKMYCr+magT74r1smjyVIwe6nb8UdNeXPlmKmtnTb/8lZLXL+N7XzNNt7kBy0Mm046BWHJP4gD8a6iszxDB9p0C9TdtxHvzjP3fmx+mK+5xUOejKPkfjODqezxEJef/AADy+iiivlD7YK9H+Cv/ACON5/2D3/8ARkdecV6P8Ff+RxvP+we//oyOtaP8RHDmf+6VPQ92ooor1T4E+WvFn/I465/2EJ//AEY1Y9bHiz/kcdc/7CE//oxqx68eXxM/R6H8KPogoooqTUK2/B3/ACPGgf8AYSt//Ri1iV2Xwphin+JmjJNGkihpXCuoIDLE7Kee4IBB7ECsMVLloTl2T/ID6jooor85JCiiigAooooAKKKKACuK+Lf/ACTDWP8Atj/6Ojrta4r4t/8AJMNY/wC2P/o6OunBf7zT/wAS/MD5fooor9EKCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAnsrSS/v7aziKiS4lWJSx4BYgDPtzW343uIpfE81tbPm0sUS0gXB+RUGCuTycNu5OfrimeCLSO98Z6ZFIWCrKZRtPOUUuPwyorHvbuS/v7m8lCiS4laVgo4BYknHtzWG9f0X5v/gG+1D1f5L/gkFFFFbmAV0/xC/5HnUf+2X/opKydAjjm8R6XFKiyRvdxKyMMhgXGQR3Fa3xC/wCR51H/ALZf+ikrnb/2iK8n+aOmMf8AZ5P+8vyZzFd38Lo5Y9W1K/Ee6GCzKsdwHzFgQPxCN+VcJXoPgbz9P8K6zqKeWyz3EFqFbJI+YKx/KUY9x+c4zWk497L72a5bFPExvstTbrzfxHEkPiC7WMYBYNjPcqCf1Jr0ivOvE/8AyMV1/wAA/wDQFrWnufUcSpfVYv8AvfozKjkkhlSWJ2jkRgyupwVI6EHsa9Q8a+fdfDHRZ28yZh9nklkOWPMRG5j7kjk9zXltew69/wAkdi/68rT+cdcmMdqlJ+Z81glelVX908eooor0Dzjp/h7/AMjzp3/bX/0U9dd8W7SR7DTLwFfLileJhnnLgEY9vkP6Vw/gy8+w+MdLl8vfumEWM4++Cmfw3Z/CvUPiFp8mqaTp1om5Vl1KFHkCbhGGDLuPtlh+YFeRipcmNpze1v8AM9jCx58DUgt7/wCRk/Fq82aXp1l5efOmaXfnpsXGMe+/9K8orp/Hev8A9u+I5fKk3WdrmGHByrY+8w5IOT3HUBa5iuzA0nSoRi9zjx9VVcRKS22PXPhlGZvBl/EpAZ7qRRnpkxpTk++Kf8KP+RWuf+v1/wD0BKdNGIb2WJSSqOyjPXANRl0v9vcf7y/M99P/AIRK6/6dy/8ASWFRXMC3VrLbuSElQoxXrgjHFS0V+gNXVmfiybTujxyirOoxJBqd3DEu2OOZ1UZzgBiBVavjpKzaPvIy5kpLqFej/BX/AJHG8/7B7/8AoyOvOK0tF17U/Dt493pVz9nneMxM/lq+VJBxhgR1AqqclGSbMMZRlWoSpx3aPquivlr/AISzxH/0MGq/+Bkn+NMl8Ta/cQvDNrmpSRSKUdHu5CrKeCCCeRXX9bXY+e/1eqfzr7h/iz/kcdc/7CE//oxqx6KK4m7u59RCPLFR7BRRRSKCvQ/gtp/234hwz+bs+xW0s+3bnfkCPHXj/WZzz0x3rzyiscRSdalKmna6sB9q0V8fWniXXtPtUtbLW9StrdM7Iobp0RcnJwAcDkk/jWjYfEHxdp07TQeIb92ZdpFxL564yD92TcAeOuM/nXzMuHauvLNfj/wRWPrCivmOH4veNop45H1ZJlVgxje1iCuAeh2qDg9OCD71r/8AC9/FH/Pho/8A35l/+OVhLIcWtrP5/wDACx9C0V4jD8f5VgjE/htHmCgOyXhVWbHJAKHAz2yfqa1LD496NJAx1HR7+3m3YC27JMpXA5yxTnOeMfjXPLKMbHVw/Ff5hY9aorzmw+NnhG8naOc39ioXcJLi3ypORx+7LHPOemOOtakPxW8EzzxwpriBpGCgvbyooJOOWZQAPckAVhLAYqO9N/cxHZVxXxb/AOSYax/2x/8AR0dbX/CY+F/+hk0f/wADov8A4qofE15a6h8PtcurK5hubd9NudksLh0bEbA4I4PII/Clh4zpV4TlF6NfmB8l0UUV+hlBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAdP4S/cWPiO9l+W3TTJLdn64kkICDHXkg89B3xXMV09j/ofw61W4+/9vvYbTb08vywZN3vnOMcetcxWFLWc5edvuS/W5vV0hCPlf72/0sFFFFbmBp+G/wDkadI/6/Yf/QxWp8Qv+R51H/tl/wCikqLwJbxXXjXTI5k3KHaQDJHzKjMp49CAag8X3cl74v1WWQKGW4aIbRxhPkH44UVy74r0j+b/AOAde2E9Zfkv+CYleg6V5+nfDmx/1bJqWsR+pKoCPyO6L34P5efV6TNbta6H4L05pfMglEt2w24+fbvX3+XzCPf0oxOrhHz/ACTZtlkW6unkvvaRp1514n/5GK6/4B/6Atei1514n/5GK6/4B/6AtaU9z6XiT/dI/wCJfkzIr2HXv+SOxf8AXlafzjrx6vdP+aXf9wX/ANo1x5hLllTf94+dy6PNGqu8TwuiiivSPLLmk3cdhrNjeShjHb3EcrBRyQrAnHvxXtfj6SSHwXfyxO0ciNEyupwVIlTBB7GvCK9tS7j1j4VyXEoaUnTXDGcbi0kakFu/O5cg9eh615WYRtUp1OzPWy6d6dWl3R4lRRRXqnknrnwnuIm0G9tg+Zo7rzGXB4VlUA5+qt+VSeL7k6ZNcXQUoqvEx2qPmUsobGfX5vxrI+FVzBaQa3Pczxwwr5G6SVwqjJcDJPvUXj3xHp+raep025jmSQrG2chsAls7TgjkAcj+leVRU4ZhKUVp3+5n02Hqw/s6cJveElb70af9r6b/ANBG0/7/AC/41OtzA9v9oSaNocE+YGBXA6nPSvIaK+sWbS6x/E/PHkcOk/wL2sT21zq91PaAiF33DPc9zz6nJ/HtVGiivKlLmk5Pqe3CKhFRXQKKKKkoKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA1n1NR4Rh0pSpY373Mg2nIAjRVwemDl/yHTvk0UVMYqOxUpOW4UUUVRJ3Hwrjjk8VzM6KzR2jshIyVO5BkehwSPxNcnq13Hf6zfXkQYR3FxJKoYcgMxIz781t+EPF/wDwiv2z/QftX2nZ/wAtdm3bu/2Tn736VzFc1OnL285taWSX6nVUqR+rwhF63bf6BXo+uPBF420XTrS78+3sbLyAokDbHUOpBxwGwq56dBXnFanh2WODXraSaRI0G7LOwAHynua0nT5pKd9r/ia5fVUK8IvrKOvoz0ivOvE//IxXX/AP/QFr0WvOvE//ACMV1/wD/wBAWlT3Pp+JP90j/iX5MyK90/5pd/3Bf/aNeF179phs/wDhBbBdQkjjs30+KOVpZNi7WQLgtkYznH41wZo7KD8z5/Kld1F5HgNFFFeqeSFek+DL/wA/4d+I7FmkZraGVxuOVVHjOAvPHKsSPf3NebVc0/U7nTGuTblcXNvJbShlyCjjB+h6H8PTIrDE0faw5Vvozow1b2M+Z7Wa+8p0UUVuc4UUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABWnoHh/UfEup/wBn6ZD5tx5byYPAAVSeT0GThQTgZYAkZrMr6C+DHhb+yvDz63dRbbvUf9VuXBSAdOoBG4/NwSCAhqZOyNaNP2krHgd3Z3VhdPa3ltNbXCY3xTIUdcjIyDyOCD+NQ19Z+KvDWh+I9LZNciQQ26vItyX8trf5eWDdgByQcrwMg4r5W1JLOLVLuPT5XmslndbeRxhnjDHaTwOSMdh9KIyuOtRdN7lWuh8GWOnalrv2TUo/MSSFvKTcwy4weq/7IbrXPVt+ELlLTxXp0kgYhpDGNvq4Kj9WFKpfkdjkr39lK29jrbn4ZWzyA2upTRR45WWISHPrkFfbtXMa/wCELzw/bC5muIJYGm8pChO48EgkEYHA9TXsdct8QbX7R4VeXft+zzJLjGd2Tsx7ffz+FcFHETc0m9DyMNjarqRjJ3TPJKKKK9I9wKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAp0kkkrBpHZ2ChQWOTgDAH0AAH4U2igAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAN3wd4cl8VeKLPS0DiF233Ei5/dxDljnBwccDIxuIHevrCGGK3gjggjSKGNQiRooVVUDAAA6ACvNPgx4W/srw8+t3UW271H/AFW5cFIB06gEbj83BIICGvTqxm7s9LDU+WF3uzxf45eJLqOS08NRLst5I1u5pA5zJ8zKqEdNoK7uc5O3pjnxeu3+Lk0svxK1NJJHdYlhSNWYkIvlK2B6DLE/Un1riK0irI4q8nKowqxYXX2HUbW72b/ImSXbnG7aQcZ7dKr0VTV9DFq6sz6ErG8V2v2zwrqMW/ZthMucZ+4d+Px24/GrukTyXWi2FxM26WW3jd2wBlioJPFSX9r9u066tN+zz4Xi3Yzt3AjOO/WvFj7svQ+Xg+Son2Z4HRRRXtH1IUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFbvg7w5L4q8UWeloHELtvuJFz+7iHLHODg44GRjcQO9YVfQXwY8Lf2V4efW7qLbd6j/qty4KQDp1AI3H5uCQQENTJ2RrRp887HpcMMVvBHBBGkUMahEjRQqqoGAAB0AFPoorA9Y+c/jVYRWnj8zxs5a8tI55AxGAw3R8e2Ix+Oa87r3L49WEsml6NqAZPJgnkgZSTuLSKGGPbEbZ+orw2t4vQ8qvG1RhRRRVGJ7L4KkeTwhYNI7OQHUFjngOwA/AACt+vO/AniDStM0mSzvbtYZpLosoZWxgqoBLYwOQeprvra7tr2MyWtxDPGDtLROGAPpkfUV5FaDjN6HzeKpyhVk2tLng93bPZXs9rIVMkEjRsV6Eg4OPyqGtfxRbPaeKNSjkKktO0g2+j/ADD9GFZFerF3imfQwlzRUu4UUUVRYUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBu+DvDkvirxRZ6WgcQu2+4kXP7uIcsc4ODjgZGNxA719YQwxW8EcEEaRQxqESNFCqqgYAAHQAV80/CTUW0/4iWC/aEhhulkt5d+AHBUlVyehLqmMck4HfFfTNZVNz0MIlythRRRWZ1nHfFLTm1L4d6qsduk00CrcJuxlAjAuwJ6EJv6ckEjvivmCvsq8tIL+xuLO6TzLe4jaKVMkblYYIyORwe1fHE0MtvPJBPG8U0bFHjdSrKwOCCD0INa03ocGMj7yYyiiitDjCrllq2o6dgWd7PAoffsSQhS3HJXoeg61TopNJ7iaTVmWL6+udSvJLu7k8yeTG59oGcAAcDjoBVeiimlbRAkkrIKKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAF3R9Q/snW7DUvK837Jcxz+Xu279jBsZwcZx1xX2HXxfX2hWVQ7sG9wooorM7Qr5J8Yf8AI7a//wBhG4/9GNX1tXyT4w/5HbX/APsI3H/oxq0p7nHjPhRi0UUVqcAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV9beD/wDkSdA/7B1v/wCi1r5Jr0fwX8XNR8OWsGmalB9v02L5UYNiaJcjgE8MoGcKcdQNwAAqJptaHRh6kYSfMfRNFc3p3j7wtqWnwXi65Y24lXd5VzcJFIh7hlJ4IP4HqCRg1a/4TDwx/wBDHpH/AIHRf/FVlZno88e5tV8heJLuC/8AFOr3lq/mW9xezSxPgjcrOSDg8jg9692+JXjjSY/A97b6Vq9nc3d5i2C2s8cpVW++WGThSgZc44LDp1HztWlNdTixc02ooKKKK0OMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAP/Z", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAGqCAIAAAAUcGK5AAAc/klEQVR4Ae3drbIkR3oG4DMbsoAHeOhIETYwn9gwEhAasESLTXwDFvIVrMD6Cox0BUsW7xKBQQOMHI7lBnaER1QCMpCBXXbPVPT0T3VVVmbll5mPQhHq012V+X1PluI92X/n6ck/BAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQIECAAAECBAgQGEzg2WD9apcAAQIFBf7pL/9wGv0f/uPXBacxdBcCn3TRhSYIECBQWWCO3sp1mL4dgV+0U6pKCRAgEFTgOn2v7wlaurLqCXgKup69mQkQaF9gOWg9Ed3+ChfswA64IK6hCRAYXGA5ngfH0b4Adg0QIECAAIEKAgK4AropCRAYR8AmeJy13tqpAN4q5ngCBAi8F1gZrisPwzqagAAebcX1S4AAAQIhBARwiGVQBAECBAiMJiCAR1tx/RIgQIBACAEBHGIZFEGAQMcCPg3c8eLuaU0A79FzLgECBAgQSBQQwIlwTiNAgAABAnsEBPAePecSIECAAIFEAQGcCOc0AgQIECCwR0AA79FzLgECQwt4d9XQy7+7eQG8m9AABAgMLCCDB178va0L4L2CzidAgMCCgIRewBn8IQE8+AWgfQIE9gosROzCQ3tndX77As/ab0EHBAgQCCFw8UcXpG+IVVEEAQIECAwicBHDg3StTQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIBAnwL/8+8v+2xMVwQINCLgm7AaWShlEiBAgEBfAgK4r/XUDQECBAg0IiCAG1koZRIgQIBAXwICuK/11A0BAgQINCIggBtZKGUSIECAQF8CAriv9dTNOgFvgV7n5CgCBAoKCOCCuIYmQIAAAQL3BATwPRn3EyBAgACBggKfFBzb0AQIEGhU4Lf/fFn4N19c3uNnAvsEnu073dkEmhQ4vQb87K++b7J6RRcSuA7di4lk8AWIH/cJeAp6n5+zCRDoQ+Bh+k5trjmmDw1dHCIggA9hNgkBApEF1ifr+iMj96u2GAICOMY6qOJAAc8/H4jdwlRbM3Xr8S0YqLGKgACuwm5SAgRiCKSladpZMTpWRRwBARxnLVRCgMCxAnL0WG+zXQgI4AsQPxIgQGCFgPBegeSQZQEBvOzjUQIE+hXY+bEiGdzvpXFMZwL4GGezECBAgACBjwQefBPWn3768qPDV//w6vnb1cc6kAABAm0KTJvgndvoNvtWdRaBBwGcPEdyci/PKNeXfTxKgMA2gSk+PZO8jczR2QRKBXC2Aj8eqESuC/WPjf1EgAABAkcILH0XdIm0O6KnAHMI9QCLcLuE+S8B+yLo20Bj3rtnE+wp6DGvmRxdN7YDztHyEWOU+N1FqB+xcuaIIHARhxIuwqKooYCAAC6AWmbI9aEuqsusgFFLClyE7vlU80OS+JzF7fYFBHD7a6gDAq0LzBFbq5H234r1qx9+uMD77sWLi3v8GE1AAEdbEfUQGEygevp26i2S4y+sL+KIv0YqJEAgsEA7v0BMkXz6N7DmWKUJ4LHWW7cECBAQw0GuAQEcZCGUkV9g/sRR/qGNSOBcoJ1N8HnV189Rnz/q9gECAvgAZFNUE5DB1ehN3IKADK67SgK4rr/ZCQwvEOTDRTvLaHMTPPzFVxlAAFdeANOXEzh915VNcDnho0cWckeLm6+sgAAu62v0CAIyOMIqLNWwc/e5NPSBj/n94EDsPqYSwH2soy4IDCBQNKfF5wBXULQWBXC0FVFPEYGLTfDNv8RwcUyROgzat4AU73t9c3fnm7ByixovqsByvi4/GrWnkeoquv0dCVKvcQTsgOOshUryC9zc6V5Pc0rflQdfn+6e4gINpa9NcPGroZ8J7ID7WUudJAjY+Cag5T9lObQaSt/8NEbsWUAA97y64/SWlqNpZ42jqlMCBIoKeAq6KK/B4wpcpK/nn6stVX/b3+WOqkGbOJyAAA63JAo6QOAifQ+Y0RSJAsIsEc5pDQgI4AYWSYkECDQm4PeGxhasTrkCuI67WQkQWCsgzNZKOa4xAQHc2IIp96aAV3BvsrizpoDfG2rqtzG3AG5jnVT5UEAGPyRq+ABh1vDiKf2ugAC+S+OBjgX++99+N/3bcYNaI0AgvoAAjr9GKlwr8HATfMrdm9HrfdFrlWsd1+ImuMWaa63vkPP6Io4hl32wpm8m7mAG2l0UkJSLPB4sJCCAC8Eatr6A3K2/BiogQOC+gAC+b+ORNgXkbkvrZuvZ0mqpNbOAAM4MariGBKbXjL30e1qvn9+8Od349PXrhlZQqQSaFhDATS+f4gmkC8yhmz7E/jOnv3RkE7yf0QhtCngXdJvrdr/qV8/f3n9wiEf+7K//bn2fD984vX6oto4Mkb5ByPwGEGQhxitDAI+35jr+WGDOYE9HfwzjJwIEygoI4LK+Rj9YYE7TTfOmnbVpCgcHFbD9DbowQ5QlgIdYZk1eC1y820gGXxP1f4/07X+NQ3foTVihl0dxRwrI4CO1P5preivW+T83c/HimPPjI9+eeglf+Q9/826Z8MW/fLZ8gEfTBO4G8J9++jJtRGcRIBBc4N47sC6eFTioi+t8mu85JfH8Y96CbsZ83ilaGO1h+k5NTMfI4BKL6SnoEqrGrCyw6Y3QlWs1/YLAFL3Sd8HnwIfW5PSB5XQy1d0dcCf9aYMAgX0C19vlOhvlfV04uy2B7p+CPX1e1A64rctStQQI7Ba4+eRzoa327mIN0LGAHXDHiztoa6f3Uv38/qsVbyPYw9126f7ehei9+VD3IBqsKiCAq/KbvJjAQsReP6darAoDhxG4l682vmGWaMBCBPCAiz56ywvZPALNWL9/3MvdaaVF7wiXe+wevQYce31UR4BAmsAUvUHSd6GMtNac1YuAHXAvK6kPAtsF0p4MmPbQaSduLzD1jIXMu7nxXTg+tQTnEXgoIIAfEjmAQFcC0bOzEPbN3C00l2EJrBMQwOucHEWAQEMC4rahxRq4VAE88OJrvZ7AT7/95Tz582/+db7tRoLAOebp9G2kAz///PvX/5UA7pRcAgI4l6RxCCQKXOfHaaCFFJlPWTgmsZpgp82d7qxrHqd7sZ1QEU7v/muwZmQBPFO4QSCWwJwZ12VNKXJ6dOGY67Pme/aHUOn3YaX1NTc43dgwwvx89cBb4XM6tw8TEMCHUZuIQDaBDelya87p9JUZXOVDwzu7u9Xx+/sedD0n8TyESJ4p3CggIIALoBqSAIGqAtkifIrkLBk8DXKd7lWJTpP/7Zs//+7FiwCFDFqCAB504bVNIKZAtuy81d714A/2xLcGcR+BXAICOJekcQgcKjAlx3WcHFpBF5M9NHzeRZuaiCngqyhjrouqCNQXWH4BePnR+tWrgEB4AQEcfokUSOCWwMOt262Tot/XZVPR0dVXT0AA17M3MwECBQS8rFsA1ZBFBARwEVaDEuheIPt3Stv+Jl8zv/rhh+RznVhRQABXxDc1gV0Cdbd6YV8DFuS7rionHyjgXdAHYpuKwCECuYJ52uMupGzeHXBjqZnr88GHXA8mCSsggMMujcIIPBCYQmvK2ncvv7o47t3Lzy/uWfnjZ9//8eLIvCl7MXihH+ffPzKH+p5v0thzbiGmqMOO80XQ0woI4KiXoboIHC5wneXHlPAXX/9nxonOusgw7E9Pf//jt///C83FLzrf/uNnX/8mY9mGGlDAa8ADLrqWuxK43rZ21Z5mCPQrYAfc1dq+ev62q340M4BA3u1vCDBPOIdYhgaKsANuYJGUSIDAVoEOc30rgePDCwjg8EukwB4F5jcK7Wwu8/uMdlbjdAIEtggI4C1ajiVAgAABApkEBHAmSMMQILBdoM8nirP8CeHtmM5oTkAAN7dkCiZAgACBHgQEcA+rqAcCBAgQaE5AADe3ZAomcCngo8CzyPsvzZh/doNAYAEBHHhxlEaAQACBlBeqvQyctHBDfQ/lJHQ7gEdTSLpUnESAQFCBlMgM2oqyeha4HcA9d6w3An0J+ChwX+upm4EEfBXlQIut1Y4Frl8GPvubBAf1fV3Dw4l/evrlw2McQKBXAQHc68rqa3SBhDicydaH955Z5uncIDCmgKegx1x3XRNYEphitelkzfte6MTRvA9r6RLz2P8JCGDXAQECBAgQqCDgKegK6KYkkFdgeh9Wrr/ukLew40ebN+5eXT4e34xbBeyAt4o5ngCB9wJz2hEhQCBBQAAnoDmFAIHRBT77+jePCbwM/Nho6CME8NDLr/laAj68W0vevATiCAjgOGuhEgKdCPT064UX1zu5KEO24U1YIZdFUQQ2Chz5Pqz5ix4X3uh0ncGSbOOSDnf4gF+BLICHu8o13KvAKfPC5tzBkTw5XM9YYemnl4G/+aLCvKZsQUAAt7BK62p89fztugMd1bPAw9SJk9APS+15nfRG4OlJALdxFQjXNtaphSpXxl6cnG4BVY0EUgQEcIpa9nPka3ZSA+4UmHL6x293juF0AgSWBATwkk6Wx4RrFsbOBonyCmVnrE9P0+8NGffueYbyMnB3l1muhgTwXkn5uldw1PNl8Kgrr28C7wVuB3ChUGnuXeaFHFx9BE4CwTN4+rhR4h8CirHAwXljIKmipsDtAC5UUd08u47/uvUUQjZsEwIr3wnVRC+KJEAgTeDQAE4rMddZ4jaXpHH2C9ic7TdMHqECvpeBk1er6xMHCuCu11FzBAhsEJgyeD564dmIvG/pmmd0g8BJQAC7EgjUEaiwD9veaOsvA6/peGUYrxnKMckC1y8RJg/V0IkCuKHFUmqfAnMALGzFCnU+T309/qmYpt+Edd3UdM9Cy/Ojxy/EzVLd2b2AAO5+iTXYjMBCNuSNhIWJ1mCdUnn+kwxrTql1zNRpXrpajZi3SwEB3OWyaqo3gYXIXBkwCyOkYc0DriwgbZZ+zvI+rH7WMlsnAjgbpYEIVBGYg3Ca/SILzx8qV9s8y8Xs5WY0MoE+BARwH+uoi/YETnE1p1eWBqbRKqbg3MvDGuYj93f9cK79UxiBQCGBXxQa17AECFQRyJJtO1NtquH0bxWBhUkTcHZSLBTjIQIC2DVAoDeBhJgpRFA9htPiMw5goXUxbBABARxkIZRBoFuB6jHcrazGGhcQwI0voPLbFDjtzJreab17+VWb9qomEEVAAEdZCXUQINC5wPRJJP8QOBMQwGcYbhIgcCbQ2ddgNf18w9mydHhzzO+hnBZSAHd4NWspuEDaO4OCNxWnvOy8TXznVxx/lawXEMDrrRxJIKdA2A1Z9gDLqWYsAh0JCOCOFlMrBBoRkPGNLJQyywr4JqyyvkYnQIBATIHvXrwIUtir52/PKxnhJeGpx6lrAXy+7m4T6EHA/vJ6FcM+4X9d6p574mTqni4u8vh6qG4SWgBfL657CBAgUEagzN9E6iN314svJ3RD8SyA1y+6IwlkEDhtT+NvyKYK3738PEPDAYbYrz29EbqzD2UFWJZSJTQUzwK41EVgXAIEDhCYwvXeU+737j+gKlOEFQgVzwI47HWiMAIVBPoLrf3b3wrLYMpKAgfHswCutM6mJUCAAIGmBLLHswBuav0VS4DACoFpH3/a+Nr+rtBySB6B5Xie5rh+d5gAzkNvFAJrBE5P8EqFNVaOIdCZwHVC+yaszpZYOwQIvBfI+4uOb4R2YWUXEMDZSQ1IgMAAAt98kdikP0qYCNfhaQK4w0XVEoGdAnn3jjuLcTqBXgUEcK8rqy8CBAgQCC0ggEMvj+IIHCnQ6IeAq+3X056FTjvryOvAXEcJCOCjpM0zvIC3QDd0CZQKdenb0EVQvlQfQypvbAYCBLoQuPGN0MuBevF+q+WDuyDSxCYBAbyJy8EECOQRmL8rI89wMUeRuDHXJUxVnoIOsxQKIUAgVeD6GeNGX89OBXBekwICuMllUzSBewL7g+cUZu9efnVvCvcTIJBFQABnYTQIgXAC15vCcCU2W9Bn3/+x2doVHkhAAAdaDKV0LBD/LdD7t84dL5/WCJQQEMAlVI1JgMDRAgfs+H/89vOjuzJf1wLeBd318mqOAIEcAh9eFxfAOTSN8UFAAH+Q8F8CBAhcCZxvrL30e8Xjjl0CAngXn5MJEOhV4Dx6e+1RX3UFvAZc19/sBHIK7H8jVdOp03TxOa8DYzUiYAfcyEIpkwABAgQOEfjyD+8/BP/212U/byaAD1lPkxB4eoq8P9u/dbbCBPoTKJ3EAri/a0ZHBAgQIJBTYE7iedAsm2MBPHu6QaAfgci77aLKU+N280WFgwz+85s3n75+XbGY80hODmNvwqq4gqYeSGDYRBxojbV6rMCUwYUmTA7UrfXYAW8VczyBPgX8itDnuvbY1Ry9pxt1t8J7gO2A9+g5l0AggT1PvfaUvj31EujyClzKFMNzJAcu80ZpAvgGirsIECBAIKbAvay9d3/MLk5VCeDIq6M2Aj0L7NmyL7vYBC/79Ppoc1thAdzrpagvAkMLyOAul3/NNrehGPYmrC6vUk2NKCByRlx1PQcQmD6SlPbGaTvgAKunBAIECBB4JLBm+/tojLWPpwXq2tE/HCeAP0j4LwECBAh0IdDKB5M8Bd3F5aYJArkFTn/79t3L999Kn3v40N+Mnb1ZA+4XWL/9bSV9JxMBvP/CMAKBbgXy/gl6r1J3e6GEaayh9J3MPAUd5sJRCIGuBaRv18tbtrmV29+20nciE8BlrxujEyBAgACBmwIC+CaLOwkQIEAghECv298JVwCHuMIUQYAAAQLJAiWefN70SaTzv064vgsBvN7KkQQIECBwqMDK7e+hNeWbTADnszQSAQIECBBYLSCAV1M5kAABAgQOFOh7+ztBCuADryZTESBAgACBDwIC+IOE/xIgQIBAGIH1298S78A6hkEAH+NsFgIECBBoTGDTG6ETehPACWhOIUCAAIGCAuu3vwWLKD+0AC5vbAYCBAgQ6F0g4aPAArj3i0J/BAgQaEpgkO3vtCYCuKkLU7EECBAg0IuAAO5lJfVBgACB9gXG2f5OayWA279gdUCAAAECDQoI4AYXTckECBDoUSDg9rfoJ5EEcI9XsZ4IECBAILyAAA6/RAokQIDAAALJ29/kE6ujCuDqS6AAAgQIEBhRQACPuOp6JkCAQCiBdnex54xbv4tDAJ/ruU2AAAECBA4SEMAHQZuGAAECBG4K9LH9vdna8p0CeNnHowQIECAQXaBohJf7JJIAjn5hqY8AAQIdCxTNzuBuAjj4AimPAAECBPoUEMB9rquuCBAgEF9g5O3vtDoCOP4lqkICBAgQ6FBAAHe4qFoiQIBAfIG829+8oyXrbfoo8CfJ0ziRAAECBAiMIHD+RuhNEbuMI4CXfTxKgAABAvkFgmxYExrLGMaegk7wdwoBAgQIENgr8GzvAM4nQIAAAQJbBMptfz99/XpLITmPnZ+aPt8iL0/gKehlH48SIECAAIHHAutzdx7LU9AzhRsECBAgUFyg3Pa3eOm5JxDAuUWNR4AAAQIEVggI4BVIDiFAgACBHAK2v+eKAvhcw20CBAgQaFigrYAXwA1fakonQIBAQwJtpeMBsAL4AGRTECBAgACBSwEBfCniZwIECBAgcICAAD4A2RQECBAg8HTMt2Q09ES3APZ/BQECBAgQqCAggCugm5IAAQIECAhg1wABAgQIEKggIIAroJuSAAECBAgIYNcAAQIECBwk4H1Y59AC+FzDbQIECBAgcJCAAD4I2jQECBAgQOBcQACfa7hNgAABAgQOEhDAB0GbhgABAgQInAsI4HMNtwkQIECgrMAx78Mq20Om0QVwJkjDECBAgACBLQICeIuWYwkQIECAQCYBAZwJ0jAECBAgQGCLgADeouVYAgQIECCQSUAAZ4I0DAECBAgQ2CIggLdoOZYAAQIECGQSEMCZIA1DgAABAgS2CAjgLVqOJUCAAAECmQQEcCZIwxAgQIAAgS0CAniLlmMJECBAgEAmAQGcCdIwBAgQIEBgi4AA3qLlWAIECBDYLeDroE+EAnj3pWQAAgQIECCwXUAAbzdzBgECBAgQ2C0ggHcTGoAAAQIECGwXEMDbzZxBgAABAgR2Cwjg3YQGIECAAAEC2wUE8HYzZxAgQIBAYIFW3mUtgANfREojQIAAgX4FBHC/a6szAgQIjCfQyvZ3WhkBPN7lqWMCBAjUFmgoJstRCeBytkYmQIAAgbsCJTK4xJh3G9j9gADeTWgAAgQIEEgSmPKyrchM6vLuSQL4Lo0HCBAgQOAAgWFj+NkBuKYgQIAAAQJrBH5+82bNYTePaW4zbQd8cx3dSYAAAQIVBJoL0QpGpiRAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQI9Cjwv4nKxMHetKhHAAAAAElFTkSuQmCC", + "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAGqAoADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5/ooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACivUPC3gzTYrC3v7xFvJp40lVZF+SMMucbckN16n0GAK5jxr4b/sa/F3b82d07FVC4ETddvAxj09gfTJwjXhKfIjlhjKc6ns0ctRRRW51BRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAfQlct8Qbr7P4VeLZu+0TJFnONuDvz7/cx+NZ3hfxzaNZwWWrTyJcr8ouJOVcZ4yex56njjJPNY3jbxVba0kVhYjfbRuJWmYEFmwRgA9AMnr1P058ynQmqqTWx4NDCVI10pLRM46iiivTPeCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqWCBriQohAIR359FUsf0FRVqaHAssl87Egw2Uzrjudu3n8GNXTjzTUTOrPkg5GXRRRUGgUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUqI0jqiKWZjgKBkk+lAbiUVpw+HtWuELpZSAA4/eEIfyYg1QuLeW1neCdCkiHDKe1K6NZ4erTipTi0n1aZHRRRTMgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiivafDFslr4Z06NCxBgWT5vV/mP6k114TC/WJON7WO/AYF4ubjzWsvU8Wor3+obm0tryMR3VvFOgO4LKgYA+uD9a7nlD6T/D/gnqPh920qfh/wTwaivb/7C0f/AKBVj/4Dp/hR/YWj/wDQKsf/AAHT/Cp/smf8yM/7Aqfzo8Qor2/+wtH/AOgVY/8AgOn+FU5/B+gXEzSvpsYZuojdkX8ApAFS8pqdJImWQVre7JfieOUV6/8A8IT4d/6B/wD5Gk/+Ko/4Qnw7/wBA/wD8jSf/ABVT/ZNbuvx/yJ/sHE/zR+9/5HkFFetz+BdAlhZEtZIWPSSOZiw+m4kfpVP/AIVxo/8Az833/faf/E1LyuutrMiWR4pPSz+Z5hRXp/8AwrjR/wDn5vv++0/+Jo/4Vxo//Pzff99p/wDE0v7MxHZfeT/YuL7L7zzCivQ5/hpC0zG31SSOL+FZIQ7D6kEZ/Ko/+FZf9Rf/AMlv/s6h5dif5fxX+Zm8oxifwfiv8zgKK7//AIVl/wBRf/yW/wDs6ZP8NJlhY2+qRyS/wrJCUU/UgnH5Unl+JX2fxX+YnlOMSvyfiv8AM4Oiuw/4VxrH/PzY/wDfb/8AxNY+u+HLzw/9n+1yQP5+7b5TE424znIHqKynha0I80o2RhUwWIpRc5waSMeiiisDlCiiigAooooAKKKKACiiigArZ0r91oms3CcSrHHEG/2XbDD8QKxq6CwgVfBWq3AJ3vLGhHbCspH/AKEa3w6vJvsn+TObFNKCT6uK/FHP0UUVgdIUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFbeneF76/gjuC0cMTnjfncV9QMflyM1Zv8AwhNaWzzxXccqxozuGUqcAZ4657+lTzI745Xi5U/aqDt/XzOboooqjgCu08Iafa/ZTf58y43FOR/q/Ye5BHPvj1zxddL4OvjFfyWbP8ky7lByfnHp2HGfyFTPY9XJZ0442HtFe+i8n0O3rH8QaMuqWZeJB9rjH7ts43Dup/XHv+NbFFYp2PvK9CFem6VRaM8joq3qv/IYvv8Ar4k/9CNVK6D8xqR5JuPYKKKKCAooq7pGkX2u6pDpumwefdzbvLj3qucKWPLEDoD3pSkopyk7JAlfRFKiuo1D4deMNM8vz9AvH8zO37MonxjHXyy2OvfGfwrnruzutPuntb22mtrhMb4pkKOuRkZB5HBB/Gs6delV/hyT9Hcpxa3RBRRRWpIUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFe36F/yL2mf9ekX/AKAK8Qr2/Qv+Re0z/r0i/wDQBXr5T8cvQ9/IP4k/Q0KKKK90+oCiiigAooooAKKKKACiiigAooooAKKKKACiiigArz/4m/8AML/7a/8AslegV5/8Tf8AmF/9tf8A2SuLMf8AdpfL80ebm/8Auc/l+aOAooor5g+KCiiigAooooAKKKKACiitPQ9J/tm9e38/ydsZfds3ZwQMdR61UISnJRjuyKlSNOLnN6IXSNOXVIby3jUm8VBLCd2FIBwyn65GPp1FaNl/yIOo/wDXwP5x1uaP4XfSNQFyl/5i7Srp5ONwPvk45AP4VY13RjqNnFbWqRxb7kSyuFAx8rZYjuelevTwc4UnJr3rNW732PDq5hTnWUVL3bp310tutfQ82orovEtrbaTFb6ZaB8HM8rvglzyF59vm44HNc7XlVabpT5Huj2qFVVoKaWjCiiiszUKKKKACiiigAooooAKKKKACiiigAooooA9I8PXiXmiW5UYMSiJh6FQB+owfxq3qEfm6bdR70TfC67nOFXIPJPpVHw1ZfYtEhy2Wm/fHngZAx+gH45rP8YahcW8ENrFlI5w29x/EBj5Rznvz+HvWFry0P0J4h0MuVSuteVfjocVRRRW5+ehWn4ekaLXrNkjMhL7cDsCCCfwBz+FZla/hj/kYrX/gf/oDUnsdWBTeKppfzL8z0Wiiiuc/TTi/GOneXcx6gg+WX5JP94Dg9e4H/jvvXL16lqVoL/Tbi2IGZEIXJIAbqDx74ry50aN2R1KspwVIwQfStoO6Phc/wfscR7SO09fn1/zEoooqzwgrtfhL/wAlO0f/ALbf+iXriq7X4S/8lO0f/tt/6JeuPMP90q/4Zfky6Xxx9T6eqlqukadrli1lqdnDdW7Z+SVc7TgjKnqrYJwRgjPBq7RX5fGTi+aLsz2Gr7nx5r9hFpfiLU9PgZ2htbuWBGcgsVVyozjHOBWfW14x/wCR31//ALCVx/6MasWv1eg3KlFvsjxZbsKKKK1EFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFe36F/yL2mf9ekX/AKAK8Qr3HRY3i0LT45EZHS2jVlYYIIUZBFevlHxy9D38g/iT9C9RRRXun1AUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFef/ABN/5hf/AG1/9kr0CvNPiRPI2s2luW/dJb71XHQsxBP/AI6PyrgzF2w8l3t+Z5mcSSwcl3t+ZxdFFFfNHxYUUUUAFFFFABRRRQAVb03UJtMvo7qEn5T8yg43r3U/WqlFOMnFqS3JlFTi4y2Z7vYeHNV1Ha0No6xnafMk+RcHoRnqPpmtxvBH2TTLm6vLvdJFA7iOEcbgCR8x6jjpgfWt/wAFXsmo+DdLu5BhpIQQPQZIA/AACofH9/Fp3gDXJ5ldla0eABACd0n7tevbLDPtmu2vmlf2jjHRJ9Ov3mGFyLCqgqk7ttX16XXZf8E+fvHUG29tLjd/rIym3HTac5/8e/SuUra1XWV1jSrb7RkX1u5XgfLIpHLexyo49/yxayxc4zrOcNmTgYTp0FCe6uvx/wAgooormOsKKKKACiiigAooooAKK9y+DnhbSr7wnc3+qaNbXM012yxyXUAcGNVXG3cCMbi4JHUjB6ccx8cZopfHNukciO0VhGkiqwJRt7tg+hwwP0I9anm1sbOi1T52zzSiiiqMQooooA9S0xGj0qzR1KssCAqRgg7RxVXxDZpeaJcBjgxKZVPoVBP6jI/GoLHxDpyWlpDPejz/ACow5YMfmKjOWxj68/Wp9X1CzXRLhvtMRE0LrGVYHecY4x15P4VhZ3P0WVbD1cJKPMmlHXVduvY83ooorc/Ogrb8JwtLr0bqRiJGds9xjbx+LCsSu38I6Uba2N/KMPOuEGCCqZ/rgH6AetTJ2R6eT4eVbGQstIu7+X/BOlooorA/RArz7xXarba47JjEyCTAXGDyD9ckZ/GvQa4jxpE41K3mI+RodoOepBJP8xVw3PD4hgpYO9tmv8jmqKKK2PhArtfhL/yU7R/+23/ol64qu1+Ev/JTtH/7bf8Aol648w/3Sr/hl+TLpfHH1Pp6iiivy49k+RvGP/I76/8A9hK4/wDRjVi16Nr/AMNfF+reJtYv7PRne2nv7h42eaOMsvmtg4ZgcHqDjkYI4Nc9rXgDxR4e046hqmkvDaqwVpFljkCk9M7WOBnjJ4yQO4r9Qw2KoOnCCmr2Wl1c8WXxM5qiiiu0kKKKKACiiigAooooAKKKKACiiigAooooAK9/rwCvf69rKPt/L9T6Th//AJefL9Qooor2j6MKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvMPiP8A8jDb/wDXov8A6G9en15h8R/+Rht/+vRf/Q3rzsz/AN3fqjyc6/3R+qOPooor5w+OCiiigAooooAKKKKACiiigD0rwj8XJfDGiDTZdGS72tlZFuTF8oVVAI2tk/L1z36VJ4v+Ln/CVeFbrSF0j7G88ke5jN5oKKdxwcLhtyp2IILdOK8xool7z5nuXGpKEOSOwUUUUEBRRRQAUUV3E3hfTZduxZYcddj5z+ea6aGFnXT5OhvRw861+XocPRXXTeELdtvkXUqeu9Q2fyxTI/CEayDzbppEPUKuwj375+nH19df7OxF7W/E0+o1r2scpRXUf8Id/wBP/wD5B/8AsqP+EO/6f/8AyD/9lU/2fif5fxX+YvqVf+X8Ue6eBJLXw18NND/ta/s7VJI/MWSWYIh81mkVctj5trcj2PXrXiHxOvLW/wDiHql1Z3MNzbv5WyWFw6NiJAcEcHkEfhWBqumtpd4IDIJAUDqwGOOnI/A1RrjlSdObUtya1SVvZtWsfRkPhXR7j4TxzwaBYy6jJoYdJEs0aVpTBkEEDJYt+Oa+c6+wNBsJdL8O6Zp87I01raRQOyElSyoFOM44yK+TNY0/+ydbv9N83zfslzJB5m3bv2MVzjJxnHTNZwe5eJhZRZSooorQ5AoqW2t5Lu5jgiGXkbaPb3PtXVQ+ELZUImuZnbPBQBRj6HNdFDC1a2sEbUsPUq/CjkKntLO4v5xBbRGSTBOBxgepJ4FdavhPT1cEyXDAHJUsMH24FPuNBjj8v+zWktJGbEkscjZCYJxjdzyFrd5bXUb6HTDATTTnt5b/AC6FW08IxWwM+q3cYjQ8qjbV6jqxx7jH61ft/EP9pazDZ2SP9n+YyylecDpj0BwBk8/N2NZOq6FLHZmaS4vL673BEIBbA68jk469+pFUtHv/AOwLx2urGXfIoGWJQquecAjnoPyrjrYapS+Nanq0cbDDVY0qcfZwbTk927efTtp3PQ6Kx9P8S2OpXi20STJIwJXzFGDjnHBPbP5VsVyNW3PraGIpV489KV0Fcp43icw2cwHyKzqTnoTgj+Rrq65/xhE8miKyjIjmVmOegwR/MinHc483hz4KovK/3O5wdFFFbn50Fdr8Jf8Akp2j/wDbb/0S9cVXa/CX/kp2j/8Abb/0S9ceYf7pV/wy/Jl0vjj6n09RRRX5ceyO7VieMf8AkR9f/wCwbcf+i2rLvPih4O0++uLK61jy7i3kaKVPs0x2spwRkJg8jtWL4l+KHg7UPCur2VrrHmXFxZTRRJ9mmG5mQgDJTA5Pevcw+DxHPF+zdrrozw5atnznRRRX6CAUUUUAei+AdNsLzQp5Lqytp3FyyhpYlYgbV4yR711X9haP/wBAqx/8B0/wrJ8B2yQeFYZFLEzyPI2exB28fgorpq+owlKPsI3S2PuMBQh9WhzRV7Gf/YWj/wDQKsf/AAHT/Cj+wtH/AOgVY/8AgOn+FaFFdHs4dkdfsaf8q+4z/wCwtH/6BVj/AOA6f4Uf2Fo//QKsf/AdP8K0KKPZw7IPY0/5V9xn/wBhaP8A9Aqx/wDAdP8ACj+wtH/6BVj/AOA6f4VoUUezh2Qexp/yr7jP/sLR/wDoFWP/AIDp/hWH4v0nTbbwtezW+n2kUq7NrxwqrD51HBArrK5/xt/yKF9/2z/9GLWOIpwVGbSWz/I5sZSprD1Gor4X08jyCvf68Ar3+vPyj7fy/U8nh/8A5efL9Qooor2j6MKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvMPiP/yMNv8A9ei/+hvXp9eSeOp5JfFlyjtlYUREGOg2hsfmx/OvNzR2oW7s8fPJJYW3do5uiiivnT5AKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvUa8ur0y2m+0WkM+3b5iK+3OcZGa9jKWrzXp+p6eWvWS9CWorkIYGZ42kCYk2p1JU5GPfIFS0V7LV1Y9Rq6sTLaXL24njtZ5ImXerRxMwZcZBGBz+FQYuP+gfqP/gDN/8AE11ngpZE8JWcUkrStE0sQdjyQsjKPwwAMdq36+Ir8U4mnUlBU1o2uvQ2hSc4KV7XR4d4i0jVb7UI5bbSdRdBEFJ+xyDnJ9V96ytJjj0zxTpw1q3MVvDdwtdRXEJOI9wLbkIyRt7Y5FfQ1FeZPO51Juc4LXs/+HOSplSnLn5tfT/gm3/wmHhj/oY9I/8AA6L/AOKr5t8fXlnqHjvV7qw+zG2ef5WtjlHIABYHAySQSSOMk4LDk+70VnHNrfY/H/gDq5Y6is5/h/wT5kor6boqv7Y/ufj/AMAw/sX+/wDh/wAE8H8KWHmXD3zj5Y/kT/eI5/IH9fauuqzqFx9q1G4mDblZztOMZXoP0xVav0PBUvZUIpqzer9f60N6NFUY8iCoZZ1iuIkeWFFcEAO2GZsjGPXqf0qatbTJWh0fVWUAkqi8+hJB/nTxVaVGnzRV3eK++SX6mqV9DJooorpAfF/rBSDULM3K263MTTFivlqwLAgEnIHToetT6d/yE7T/AK7J/wChCu5v/wDkH3P/AFyb+Rr5HiHEexxEFa91+p6WCxEoR5I9ziKx/FLqvh64DMAWKBQT1O4HA/AH8q0rvzvsc/2f/X+W3l9PvY468da83vdZv9QhEN1P5iBtwGxRz07D3NcsI3dzTOcfDD0XSkneaaXbsUaKKK2PgQr0L4L6f9t+IcU/m7PsVtLPt2535Ajx14/1mc89Md689r0/4E/8jve/9g1//RkVefmsnHBVWuzNaKvUR9CUUUV+ZHrnyN4x/wCR31//ALCVx/6MasWtrxj/AMjvr/8A2Erj/wBGNWLX6vh/4MPRfkeLL4mFFFFbEhRRRQB7D4MjePwlYK6MpIdsMMcF2IP4gg1vVn6F/wAi9pn/AF6Rf+gCtCvr6K5aUV5I/QcNHlowj2S/IKKKK1NgooooAKKKKACuf8bf8ihff9s//Ri10Fc/42/5FC+/7Z/+jFrDE/wZ+j/I5sZ/u1T/AAv8jyrTbZLzVbO1kLBJp0jYr1ALAHH517rXiGhf8jDpn/X3F/6GK9vrzsoXuSZ5GQJezm/NBRRRXsH0AUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFeQeNv+Rvvv+2f/ota9fryDxt/yN99/wBs/wD0WteXm38Fev6M8TPv92j/AIl+TOfooor58+TCiiigAooooAKKKKACiiigAooooAKKKKACiiigAr0fTP8AkFWf/XBP/QRXnFemW0P2e0hg3bvLRU3YxnAxXr5SnzSZ6WWr3pMlooor2z1hmjeNLHwut7p1+t7O32ppozEqlURwrbRlhjkt+ee9dJH8RfDEkSO2oNGzKCUaCTKn0OFIz9Ca8v8AFdpKL1bpIT5RjAeRRxuyRz+Y61ztfC4/KqLxE27q7uebLMK+Hk6aSstrnun/AAsLwt/0FP8AyXl/+Jq1YeMvD2p3S21rqcbTN91XVo9xyAACwGTkjgc14DWn4b/5GnSP+v2H/wBDFcE8qoqLab/D/Iqnm1aUkml+P+Z9EVia14s0fQJfJv7lluDF5qxLGzFxyBg4xkkEckVsySLFE8jnCIpZj6AV4543tdQ1zxPNc2yedaKiRwNlVwoGSMHB+8W6/wAq4cvwE8XN2i3FdketjK86NO9NXZ1f/C19C/59NR/79p/8XR/wtfQv+fTUf+/af/F15ofDeqgKRbg5GSBIvHPQ8/5zVOXTb6Hf5lnOAmdzbDgY6nPTHvXsyyWEfig19548swxkd1+B6PRRRX3x6QVrRFk8LzlIwfMuAsjY5CgAjn6/zrJrHutQuYfE1paC7ljtJI8vD5hEbN82MrnBOQPyFcWOjzRh5Si/xJlVVPV9dPvNiiiiu0ofC0iTxtDnzQwKYGTnPHFd/f8A/IPuf+uTfyNcFaTLBqFm7AkG5iTj1ZwB/Ou41aQxaVcMoBJXbz6E4/rXxXFD5sTRgt/82v8AI68JrK3ocbXlupW4tdTuoFQoqSsFU54XPHX2xXqVcH4vtvJ1kTAPieMMSem4cYH4AfnU03qVxJR5sPGovsv8H/SOfooorU+KCvT/AIE/8jve/wDYNf8A9GRV5hXp/wACf+R3vf8AsGv/AOjIq83N/wDcavoa0P4iPoSiiivzQ9c+RvGP/I76/wD9hK4/9GNWLW14x/5HfX/+wlcf+jGrFr9Xw/8ABh6L8jxZfEwooorYkKKKkggkubiK3hXdLK4RFzjJJwBzQlfRDSbdke7wQR21vFbwrtiiQIi5zgAYA5qSiivs0raI/RkklZBRRRTAKKKKACiiigArn/G3/IoX3/bP/wBGLXQVz/jb/kUL7/tn/wCjFrDE/wAGfo/yObGf7tU/wv8AI8q025Sz1WzupAxSGdJGC9SAwJx+Vei/8LH0f/n2vv8AvhP/AIqvMKK+boYupQTUOp8dhcfWwyap9T0//hY+j/8APtff98J/8VR/wsfR/wDn2vv++E/+KrzCit/7TxHdfcdP9tYvuvuPV7Xx/oVxv8ySe224x5sRO76bc/r61Y/4Tbw7/wBBD/yDJ/8AE15BRVrNa6Wy/r5miz3EpWaT+T/zPX/+E28O/wDQQ/8AIMn/AMTR/wAJt4d/6CH/AJBk/wDia8goo/tat2X4/wCY/wC3sT/LH7n/AJnr/wDwm3h3/oIf+QZP/iakg8YaBcTLEmpRhm6GRGRfxLAAV45RTWbVuqX4/wCYLPsRfWMfx/zPZ7rxVoVps8zU4G3Zx5RMnT125x+NVZPHHh9I2Zb1nIBIRYXy3sMgD8zXkVFDzat0S/H/ADHLPsQ3pFfj/men/wDCx9H/AOfa+/74T/4qqtz8S7ZZALXTZZUxyZZAhz9AG/nXnVFZPMsQ+v4GEs5xbWjS+SO8n+JczQsLfS445f4WkmLqPqABn86p/wDCx9Y/59rH/vh//iq4+is3jsQ95GMs0xcndzf4HST+OtflmZ0uo4VPSOOFSo+m4E/rVW58W69dRhJNSlUA5zEBGfzUA/hWLRWTxNZ7zf3mMsZiJbzf3s0P7d1j/oK33/gQ/wDjVGSR5ZGkkdndyWZmOSSepJptFZucpbsxlOUvidwoooqSAooooAKKKKACiiigAooooAKKKKACiiigAooooAK9IsJ5rizjkniEchHIVgyt7ggng/59a83roZdVvNM0zTI45cSFDI6MN2UJ+Tn0xnoa9DAV1RcpS2t+p24OqqTk5bHYUVxn/CW3/wDzxtv++W/+Kq1/wmP/AE4f+Rv/ALGvUWY4d7u3yPQWOovqdTRWPYeI7O9mjgKvFK44342lvQH/APVSaj4jt9PuJbfyJZJkxxwFOQD169D6Vt9ao8nPzaGv1ily899DQbT7J3LvZ27MxySYlJJ/KpbSy0+3vrac2cK+TMkmY4wGG1geCPpXH3Xiq+mJECpAucjA3N06Enj9KsaJrGpXuqxQyzGSLDFwI1GBg4yQOOcVxvFYWq/Zct+bTbuc8cXQlNRSvfyPbdR/5Bl3/wBcX/8AQTXn1eiyQLLavbktsZChOcnBGOp715HeeIrWz1m9sZoZESC4eFZAQ2QrEZPTHTtmvnuF8VRpRqU5Ozbueli6kadnJ2ua9FVrXULS9ANvcJIcZ2g4YDOOR1FLd3tvYxCW5k2IW2g7Sefw+lfZe0jy819DHnjbmvoWKKy/+Ej0r/n6/wDIbf4Uf8JHpX/P1/5Db/Co+s0f5196I9vS/mX3mpXM+JPLttV069bccN8wHorA8e/JrXtta0+7uFgguN8jZwuxhnAz3FYvjH/ly/4H/wCy1zYypGWHlKDva35owxU4yoOUXe1vzOoZlRC7sFVRkknAApaxtUvHPhczllSSeJOB33YyBn2zXGQ3Nxb7vInli3ddjlc/lSxGPjRmo2vdXFWxipSSte6ueiyzNHqGkoAMS6jbo2fTeDx+Qr0HW/8AkET/APAf/QhXm2hTT31ppc0pMkrXkJYhewnHYegFek63/wAgif8A4D/6EK+U4glzY2jNbNJ/ierl75pqXR2OQrmvGdt5mnwXADkxSbTjoFYdT+IA/GulrO16D7RoV4m7biPfnGfu/Nj9KcXZntZjS9thKkPL8tUeaUUUV0H5qFen/An/AJHe9/7Br/8AoyKvMK9P+BP/ACO97/2DX/8ARkVebm/+41fQ1ofxEfQlFFFfmh658jeMf+R31/8A7CVx/wCjGrFra8Y/8jvr/wD2Erj/ANGNWLX6vh/4MPRfkeLL4mFFFFbEhWhoX/Iw6Z/19xf+his+t7wZGkni2wV0VgC7YYZ5CMQfwIBrWiuarFeaNsNHmrQj3a/M9hooor68/QQooooAKKKKACiiigArn/G3/IoX3/bP/wBGLXQVz/jb/kUL7/tn/wCjFrDE/wAGfo/yObGf7tU/wv8AI8gooor5I+BCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAkgha4uI4UIDSOEBPTJOKva7Ij6o8URzDAqwxjH3Qo5Hvzmm6FCs+t2qMSAH38eqgkfyqnPM1xcSTOAGkcuQOmSc1ttR9X+X/Dmu1L1f5f8OR0UUViZBWp4j/5D1z/wH/0EVU09VfU7VHUMrTICCMgjIq34j/5D1z/wH/0EV0JfuG/NfkzZL9y35r8mZddN4ItXudaIHyoyiIt1wWYY4/A/lXM11/gjzLeZLldpEl7bxAH/AHwD+j1NBuMnNdE39yZpg0nWjfoe2V4N47t4rXxrqccKbVLrIRkn5mRWY8+pJNe814X8Qv8AkedR/wC2X/opK+Yyl/vmvL9UeznC/cJ+f6M5lWZHDoxVlOQQcEGuq1zzJfC1jIdzn92zseeqHkn6n9a5Suz1D/kTE/64Q/zWvsMIr06q8jyMMrwqLyOMooorgOM1PDn/ACHrb/gX/oJrY8YQs1vazAjajshHfJAI/wDQTWFok3ka1aPt3ZfZjOPvfL/Wuq8R27XVnbQjIDXKKzAZ2g5GT+Yr1cMufBzit7/5Ho0FzYWcfP8AyKni+bbaW0G377l92emBj/2b9K5GtTX9Q+36k2xswxfImDwfU9ccnv6YrLrkxtVVK8pLY5sVUU6ra2O48DyrBbLMwJWO7DEDrgBTXqet/wDIIn/4D/6EK8n8Jf8AIKl/67n/ANBWvUtQlafw55zABpI42IHTJINefnVL3cLU+X4q36n0uUP3YL0OWqO4hW5tpYHJCyoUJHXBGKkoqD6lpSVmeR0VZ1CJINSuoYxtRJnVRnOACQKrV0n5XOLhJxfQK9P+BP8AyO97/wBg1/8A0ZFXmFamheItV8M3z3ukXX2a4eMxM/lq+VJBIwwI6qPyrkx1CWIw06UN2uo6clGakz6/or5G/wCEx8T/APQx6v8A+B0v/wAVTJvFfiK4gkgn1/VZYZFKPG95IyspGCCCeQR2r5ZcLVf+fi+5nZ9cj2H+Mf8Akd9f/wCwlcf+jGrFoor7GnDkgo9kcLd3cKKKKsQV1HgC1+0eKEk37fs8TyYxndkbMe33s/hXL0VpSmqc1Nq9jahVVKrGo1ezue/0V4ZBq2pW0Kw2+oXcUS/dSOZlUd+ADVq28T63ayF49TuWJGMSv5g/Jsj8a9lZtDrFn0Uc/pfagz2mivIo/HHiBJFZr1XAIJRoUw3scAH8jV7/AIWPrH/PtY/98P8A/FVqs0oPe6N455hXvdfL/gnp9FefR/ExxGok0pWcAbis+AT3wNpx+Zq3bfEmwaMm6sbmJ88CIq4x9SV/lWqx+Hf2vzN45rg5ac/4P/I7aiuVtviDok8hWQ3NuAM75Ysg+3ykn9KuR+M/D8kiouoqCxABaJ1HPqSMD6mtViqD2mvvN447DS2qL70b1c/42/5FC+/7Z/8Aoxa0P7d0f/oK2P8A4EJ/jUerzw3PhjUZreWOWJrSXa8bBlPykcEU6rjOnKKfRjryhUozhFrVP8jxSiiivkj4EKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigDU0j93BqU7cRi1aMt/tMQFH44rLrUg/ceG7uT732idIcdNu35s+/pWXW1XSEF5fm3+hrU0jFeX6hRRRWJkWtM/wCQrZ/9d0/9CFWvEf8AyHrn/gP/AKCKboEaS65aq4yAxbr3AJH6gVHrMzT6xduwAIkKcei8D+VdW2G9Zfkv+CdH/MP8/wBCjXX+G/MtrfSPulbnVYfqAHH65T8q5Cu+8NWqnVvC9nKfMiZpLnHT51Qup/An8axT5KNWb6Rf46GmCi3P7l97SPXa8L+IX/I86j/2y/8ARSV7pXhfxC/5HnUf+2X/AKKSvmsp/jv0/VHsZx/AXr+jOYrs9Q/5ExP+uEP81rjK73/mVf8Atx/9kr7LALmjUXkeTg1dTXkcFRRRXnHCTWcy297BM4JWORXIHXAOa7jxAzJolw6MVZShBBwQd4rga7lZlvfCbSOC/wDozZL8ksoIz+YzXp4CV6dSn3R34OV4Th5HDUUUV5hwHYeEZEOnzxA/Osu4jHYgY/ka7zUdQNr8PnufMLGF4w+MMQvnKCOf9k/yrznwlLHDHfSSuqIPLyzHAH3u9b2va5aN8Pzb2l5azPeXQDxB/wB4iDnO3gj5kHUYwfeunMFGpgaKb1Uk/uuj3sLiPZYfmTs7O36Fz+1dO/5/7X/v8v8AjVhbiF4PPSaNocE+YGBXA6nPSvJ6K4vZnVHiap9qmvvLurzW9zq1zNag+S77hnue5/E5P41SoorQ+aqTdSbm+ruFFFFBAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBbN0P7HS0BGTcNKwwcj5QB/7N+VVKKKqUnLcbk3uFFFFSI3fCaq2rOSoJWElSR0OQOPzNZF5MtxezzICFkkZwD1wTmr2j6x/ZPnfuPN83b/HtxjPsfWsuumc4+whBPXVv9Dec4+yjFPXW4V6BoBjT4iaLbQzeZHBA8eN2dpEbg5x34Ga8/rovAtzBaeMrCe5njhhXzN0krhVGY2AyT71yV5tYapBLdf5l4Spy1Iru1+Z7zXhfxC/5HnUf+2X/opK90rwv4hf8jzqP/bL/wBFJXgZT/Hfp+qPazj+AvX9GcxXe/8AMq/9uP8A7JXBV6Fa+T/YNuLllWFrdFcs20YKgdfxr7TLVdzXkeXgFdyXkee0UUV5h54V02iXG/w3qUBLExI7DPQBlPA/EH865mpra6ltTKYyP3sbRMCOqkVvh6vsp8z80bUans5XIaKKKwMQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACtPQPD+o+JdT/s/TIfNuPLeTB4ACqTyegycKCcDLAEjNZlfQXwY8Lf2V4efW7qLbd6j/qty4KQDp1AI3H5uCQQENTJ2RrRp+0lY8Du7O6sLp7W8tpra4TG+KZCjrkZGQeRwQfxqGvrPxV4a0PxHpbJrkSCG3V5FuS/ltb/AC8sG7ADkg5XgZBxXytqSWcWqXcenyvNZLO628jjDPGGO0ngckY7D6URlcdai6b3Ktdf8NtL0bWvFX9n61D50ctu/kR7nXMgIbqpH8Ifrx+OK5Cum+Hl7Hp/j7R5pVdlaYwgIATukUoPwywz7VrTtzq5wYxSeHnyuzs7WPQb34IWUkwNhrVxBFtwVnhWVi3POQV46cY/GuH8W/DzUvCNmL65u7Se1e48iMxlg5yGIJUjA4X1P49a+jq4T4u2P2vwHLP5mz7HcRz42535Pl49v9Zn8Peu2rQgotpHzOAzXEyrwp1JXTdtj59ooorzz68KKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigApWZnOWYscAZJzwBgUlFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAbvg7w5L4q8UWeloHELtvuJFz+7iHLHODg44GRjcQO9fWEMMVvBHBBGkUMahEjRQqqoGAAB0AFeafBjwt/ZXh59buott3qP+q3LgpAOnUAjcfm4JBAQ16dWM3dnpYanywu92eL/HLxJdRyWnhqJdlvJGt3NIHOZPmZVQjptBXdznJ29Mc+L12/xcmll+JWppJI7rEsKRqzEhF8pWwPQZYn6k+tcRWkVZHFXk5VGFXNJvv7L1mx1Dy/N+y3Ec/l7tu7awbGecZxVOiqTsYyipJpn17XOePrH+0fAesweZ5e23M+duc+WRJj8duPbNaPh66mvvDWlXdy++eezhlkfAG5mQEnA4HJqbVrH+1NGvtP8AM8r7VbyQeZt3bdylc44zjNew/ej6n53Tbo1k2/hf5M+TqKKK8c/RQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK3fB3hyXxV4os9LQOIXbfcSLn93EOWOcHBxwMjG4gd6wq+gvgx4W/srw8+t3UW271H/VblwUgHTqARuPzcEggIamTsjWjT552PS4YYreCOCCNIoY1CJGihVVQMAADoAKfRRWB6x85/GqwitPH5njZy15aRzyBiMBhuj49sRj8c153XuXx6sJZNL0bUAyeTBPJAykncWkUMMe2I2z9RXhtbxeh5VeNqjCiiiqMT6R+Gcsk3w80l5ZHkYLIgLsSQqyOAPoAAB7Cusrx74VeLtC0TQJdO1LUEtrma+Loro23aVRQSwG0DIPUjGK9YstQstShM1heW91ErbC8Equobg4yD15H516tGacEfB5jQnTxE207NvW2mp8qahZSabqd1YTMjS20zwuUJKllJBxntxVaug8dWUmn+OdZhlZGZrp5gUJI2yfOPxwwz71z9eXJWbR9xRnz04z7pMKKKKRoFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAbvg7w5L4q8UWeloHELtvuJFz+7iHLHODg44GRjcQO9fWEMMVvBHBBGkUMahEjRQqqoGAAB0AFfNPwk1FtP+Ilgv2hIYbpZLeXfgBwVJVcnoS6pjHJOB3xX0zWVTc9DCJcrYUUUVmdZx3xS05tS+HeqrHbpNNAq3CbsZQIwLsCehCb+nJBI74r5gr7KvLSC/sbizuk8y3uI2ilTJG5WGCMjkcHtXxxNDLbzyQTxvFNGxR43UqysDggg9CDWtN6HBjI+8mMooorQ4wrS0zxBrGjbRp2p3dsiyCXy45SELccleh6DqDnFZtFNNrYmUYyVpK6Lmqapea1qMuoahN511LjfJtC5wAo4AA6AVToopN31Y4xUUklZIKKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAF3R9Q/snW7DUvK837Jcxz+Xu279jBsZwcZx1xX2HXxfX2hWVQ7sG9wooorM7Qr5J8Yf8AI7a//wBhG4/9GNX1tXyT4w/5HbX/APsI3H/oxq0p7nHjPhRi0UUVqcAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV9beD/wDkSdA/7B1v/wCi1r5Jr0fwX8XNR8OWsGmalB9v02L5UYNiaJcjgE8MoGcKcdQNwAAqJptaHRh6kYSfMfRNFc3p3j7wtqWnwXi65Y24lXd5VzcJFIh7hlJ4IP4HqCRg1a/4TDwx/wBDHpH/AIHRf/FVlZno88e5tV8heJLuC/8AFOr3lq/mW9xezSxPgjcrOSDg8jg9692+JXjjSY/A97b6Vq9nc3d5i2C2s8cpVW++WGThSgZc44LDp1HztWlNdTixc02ooKKKK0OMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAP/Z", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAGqCAIAAAAUcGK5AAAc+klEQVR4Ae3dr64k150H8DuR1yAmA5Z4IsVg8UYjowCjAUuSxxiyaF/AMhjlBYxM/AahNgkYZBA0inbxgkSKhyxwwCzwgmw5PVNud3VXV506/88nGsl9u6rOOb/Pqeh7T3V13YcH/yNAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgMBgAo8Gq1e5BAgQSCjwyVe/ObX+zW+/TtiNprsQeK+LKhRBgACBwgJz9BYeh+7bEfhZO0M1UgIECFQqsEzf5TuVDt2wygm4BF3OXs8ECLQvsB60LkS3P8MJK7ACToiraQIEBhdYj+fBcZQvgJ0DBAgQIECggIAALoCuSwIExhGwCB5nrvdWKoD3itmfAAECbwU2huvG3bCOJiCAR5tx9RIgQIBAFQICuIppMAgCBAgQGE1AAI824+olQIAAgSoEBHAV02AQBAh0LODbwB1P7pHSBPARPccSIECAAIFAAQEcCOcwAgQIECBwREAAH9FzLAECBAgQCBQQwIFwDiNAgAABAkcEBPARPccSIDC0gLurhp7+w8UL4MOEGiBAYGABGTzw5B8tXQAfFXQ8AQIEVgQk9ArO4JsE8OAngPIJEDgqsBKxK5uO9ur49gUetV+CCggQIFCFwMUfXZC+VcyKQRAgQIDAIAIXMTxI1cokQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECHQq8OKPnRamLAIE2hDwJKw25skoCRAgQKAzAQHc2YQqhwABAgTaEBDAbcyTURIgQIBAZwICuLMJVQ4BAgQItCEggNuYJ6MkQIAAgc4EBHBnE6qcbQJugd7mZC8CBNIJCOB0tlomQIAAAQI3BQTwTRobCBAgQIBAOoH30jWtZQIECDQq8O2Hv7kY+ZPXX1+840cCBwUE8EFAhxMg0InAMnTPC5u2yuBzEK+PC7gEfdxQCwQINC+wnr6n8rbs0zyEAjIKCOCM2LoiQKBKge3Jun3PKgs1qLoEBHBd82E0OQRO30H67Nc5+tJH9QJ7M3Xv/tUDGGAxAQFcjF7HBAgUFwhL07CjihdrALUJCODaZsR4CBDIJCBHM0Hr5oaAAL4B420CBAjcFhDet21s2SoggLdK2Y8Agc4EDn6tSAZ3dj7kL0cA5zfXIwECBAgQeLjzII7vPv42DOnxqydhBzqKAAECrQhMi+CDy+hWKjXOFAJ3Aji4y+DkXu9Rrq/72EqAwC6BKT5dSd4lZueIAqkCOOIQz5tKketC/VzYawIECBDII7AWwCnSLk9Vu3pJUaZQ3zUFWXf2l4CzcjfQmUVwA5PU6RDXArjTknOUJdRzKOujU4GLa8I+ZO10npV17yYsQvUIbA916+96Zs1INgpchO75UfMmSXzO4nUHAlbAHUyiEgi0LTBHbKkyOrgK/fc/f3ih9+ij1xfv+LE2AQFc24wYD4GxBIqnb6/cIrn+mfUgjvrnyAgJEKhXoKFfIKZIPv2rV3OwkQngwSZcuQQIDC8ghis5BQRwJRNhGAkEfOMoAaomlwINLYLPB7+8Rn2+1esMAgI4A7IuygnI4HL2eq5fQAaXnSMBXNZf7wRGF6jky0UHh9HoInj0k690/QK49AzoP53AZ7/+oW2L4HTCeVsWcnm99ZZcQAAnJ9ZBeQEZXH4O1kZwcPW51nTGbX4/yIjdSVcCuJOJVAaB7gWS5rT47P78qbBAAVzhpBhSAoGLRfDp6vRFPxf7XGz1I4F7AlL8npDtPxHwJKyfcPihZ4H1fF3f2rNLG7UlXf62QWCU3QlYAXc3pQo6F7i60j3f4fT6lL4bd14e7p3EAg2lr0Vw4nOhq+atgLuaTsXsFrDw3U0W/4D10GoofePTaLFrAQHc9fSOU1xYjoYdNY6qSgkQSCngEnRKXW3XLHCRvq4/F5qs/pa/6xUVYtZtjQICuMZZMabkAhfpm7w/HQQKCLNAOIe1ICCAW5glYyRAoCkBvzc0NV3FBiuAi9HrmACBLQLCbIuSfVoUEMAtzpoxLwR8grsg8UZZAb83lPVvoncB3MQ0GeQGARm8AanRXYRZoxNn2OsCAnjdx9Y+Bb7/5H+nf33WpioCBBoREMCNTJRhbhG4twg+5e716HVf9Bbhcvu0uAhucczlZnjEnj2IY8RZH63m64k7moJ6bwtIyts2tiQUEMAJcTVdVkDulvXXOwEC6wICeN3H1vYE5G5Dc2bp2dBkGWp0AZ8BRyfVYDsC9z4zbqeSoyP9/uXL07+jDTmeAIHNAlbAm6nsSKAvgSlxixc0/aUji+Dis2AApQSsgEvJp+r38asnqZpupN33v/n5jpGOugiuIX13TFPKXf0GkFJX22sCAnhNx7YhBOYM9k2kIeZbkQRqERDAtcyEccQRmNN0V3NhR+3qws5VClj+VjktowxKAI8y0+q8EHj/2bOfvCODf8IxxA/Sd4hprrhIN2FVPDmGlllABmcGf9fddCvWu5c//PdqLl7sc75/za+nWuof+X+++WTd8FcffLO+g61hAjcD+LuPvw1r0VEECFQucOsOrMurAlnKWObT/M4piecf4w7naszH7aKJ1u6m71TFtI8MTjGbLkGnUNVmYYF9N0IXHqzubwpM0St9b+rk3bAlp/OOqIfebq6AeyhODQQIHBZYLpeLLJQP16GBlgS6vwR7+r6oFXBLJ6WxEiBwXODqxedES+3jo9VCxwJWwB1P7qilne6lWn3MkzXcmCfHSvRe3TSmkqqzCQjgbNQ6yiqwErHLa6pZR6azEgK38tXCt8Rs6POtgAB2KgwnsJLNI1gM9fvHrdydJlr0jnC2V16jz4ArnyDDI0AgRGCK3krSd2UYIYU5piMBK+COJlMpBHYKhF0MmNbQYQfuHF347iuZd3Xhu7J/+CAcSeCegAC+J2Q7gb4EKs/ORNhXczdRX5olsFFAAG+EshsBAs0IiNtmpmrsgQrgsedf9YUEPv/lV3PP//GX386vvQgQOMc8Hb6LdOTrz//1z/8SAO6QWAICOJakdggECizz49TQSorMh6zsEziayg6bKz04rrmd7sUOQtVwePePwZqRBfBM4QWBugTmzFgOa0qR09aVfZZHze8cD6HU92GF1TUXOL3Y3sJ8vXrkpfA5ndfZBARwNmodEYgmsD1drnY5Hb4xg4t8afhgdVdLPr25XvWcxHMLInmm8CKFgABOoapNAgRKCsSK8CmSo2Tw1Mgy3UsCvev7X//nvx999PrdT/6bW0AA5xbXHwECKwKxsvNqF8vG19fEVxvxJoFYAgI4lqR2CGQVmJJjGSdZR9BFZ/cN/++LLgpVRI0CHkVZ46wYE4EaBNY/AF7fWsP4jYFA5QICuPIJMjwC1wXuL92uH1f1u10WVbW4wRUVEMBF+XVOgEBsAR/rxhbVXioBAZxKVrsE+haI/kxpy9/gE+bvf/4w+FgHFhQQwAXxdU3gkEDZpV61nwEL8kNnlYMzCrgLOiO2rghkEYgVzNMadyVl466A20rNWN8PznI66KReAQFc79wYGYF1gSm0pqx98+LpxW7PHy7fudjh1o8ffPani01xU/ai8UQ/zr9/xA31I0/SOHJsIqVqmx3nQdDTFAjgas9DAyOQW2CZ5XlG8OWXv4vY0VzFl19GaPX3//Tvz59/OjX05vLXmr/+7YtfROhAEwML+Ax44MlXehcCy2VrF2UpgkD/AlbAXc3x41dPuqpHMQMIxF3+1gDmgnMNs9DEGKyAm5gmgyRAYJ9Af7m+r357tyAggFuYJWPsTmC+UehgZXHvMzo4GIcTILBLQADv4rIzAQIECBCIIyCA4zhqhQCBAIEuLxRH+RPCAZgOaU5AADc3ZQZMgAABAj0ICOAeZlENBAgQINCcgABubsoMmMClgK8CzyKnh2bMP3pBoGYBAVzz7BgbAQLlBQI+qPYxcNi0DfUcyonoegCPphB2rjiKAIE6BQIis85CjKpvgesB3HfNqiPQk4CvAvc0m2oZSsCjKIeabsV2K7D8GHj+mwTZal6O4W7Xn//y7i52INCtgADudmoVNrhAQBzOYtvD+0gvc3deEBhTwCXoMedd1QTWBKZYbTpZ494LHdaa+7DWzjDb/iEggJ0IBAgQIECggIBL0AXQdUkgrsB0H1asv+4Qd2D5W5sX7j5dzo+vx70CVsB7xexPgMBbgTntiBAgECAggAPQHEKAwOgCf/viF3cJfAx8l2jwHQTw4CeA8ssI+PJuGXe9EqhJQADXNBvGQqALgZ5+vfDhehenZKVFuAmr0okxLAK7BHLehzU/6HHlRqdlBkuyXRM64M4DPgJZAA94niu5T4FT5lWbc5kjeXJY9ph/4qePgZ+8/jp/v3psQkAANzFNmwb5+NWTTfvZqWuBu6lTT0LfHWrXE6U4Ag8CuI2TQLi2MU8tjHJj7NWT0y2gGiOBEAEBHKIW/Rj5Gp1UgwcF/pHTnx5sxOEECKwICOAVnDibhGscx75aqeQTyr5Qf6hm+r0h4to9SlM+Bu7vNItVkQA+KilfjwqOerwMHnXm1U3grcD1AE4UKs3dZZ7IwdlH4CRQeQZPXzcK+0NAlcxv5byVKBlGQYHrAZxoQGXzbBn/ZceTCFmzTQhsvBOqiVoMkgCBMIGBnoQ1xe3FvzAyRxE4LhDlw8Xjwxizhfz4Hgo95pl2t+qsK+C7o7EDAQIEMgicZ/DK1Yi4t3RlqEsXbQkMtAJua2KMtnuB8wyottj5qZPVjvD4wKaJmP8db00LYQLLjwjD2mnrKCvgtubLaDsUmJN4ZSmWqOy562X7p8E0fRPWsqjpnZWS5635J+LqUL3ZvYAA7n6KFdiMwEo2xI2ElY62YJ1SuYnF8VRpXLotPvYhsFFAAG+EshuBkgIrkbkxYFZaCCtsbnDjAMJ66eYoj+PoZiojFiKAI2JqikABgTkIp74vsvB8U7qRzb1c9J6uRy0T6ENAAPcxj6poT+AUV3N6RSlgaq1gCs613B3DvOfxqu/2dbwLLRBIJOAu6ESwmiVQRiBKth1MtWkMp39lCG73GoBzkOL2WGwh8CCAnQQEehMIiJlEBMVjOCw+6wFMNC+arURAAFcyEYZBoFuB4jHcrazCGhcQwI1PoOG3KXBamTW90nrz4mmb9kZNoBYBAVzLTBgHAQJ9C3gidN/zG1CdAA5AcwiBIQQ6ewxW09cb+j7hxnwO5TSnArjvE1t1NQqE3RlUYyVVjik6bxPP/KpyKgzqjoAAvgNkM4FEAtUuyKIHWCJAzRJoXUAAtz6Dxk+gPQEZ396cGXECAU/CSoCqSQIECFQv8Oij15WM8fGrJ+cjGeEj4anGqWoBfD7vXhPoQcD6cjmL1V7wXw71yDv1ZOqRKi7yeNlUNwktgJeT6x0CBAgkEUj0N5H6yN3t4usJ3VA8C+Dtk25PAhEETsvT+hdk0wifPzyNUHAFTRzXnm6E7uxLWRVMS6ohNBTPAjjVSaBdAgQyCEzheuuS+633M4xKF9UKVBXPArja88TACBQQ6C+0ji9/C0yDLgsJZI5nAVxonnVLgAABAk0JRI9nAdzU/BssAQIbBKZ1/Gnha/m7QcsucQTW43nqY3l3mACOQ68VAlsEThd4pcIWK/sQ6ExgmdCehNXZFCuHAIG3AnF/0fFEaCdWdAEBHJ1UgwQI9C/w5PXXYUX6o4Rhbl0eJYC7nFZFETgkEHfteGgoDibQr4AA7nduVUaAAAECFQsI4Ionx9AI5BVo9EvApdbrYVehw47KeyLoLZOAAM4ErRsCboFu6BxIFOrSt6FzIMNQfQ0pA7IuCBDoQWD5ROj1QL2432p95x6A1LBTQADvBLM7AQIxBOZnZcRorNI2JG6lE1PNsFyCrmYqDIQAgVCB5RXjRj/PDgVwXJMCArjJaTNoArcEjgfPKczevHh6qwvvEyAQRUAAR2HUCIHqBJaLwuqG2OyAPvjsT82O3cArEhDAFU2GoXQsUP8t0MeXzh1Pn9IIpBAQwClUtUmAQG6BDCv+588/zV2V/roWcBd019OrOAIEYgic0v35w9MYjWmDwFsBAexUIECAwE2B84W1j35vMtkQJCCAg9gcRIBA7wLn0dt7reorI+Az4DLueiWQQuD4jVRNp07Tg09xPmizcgEr4MonyPAIECBAIKvAv3333am/Pzx+nLRjAZyUV+MEfhSoeX12fOn8Y51eEehFIHUSC+BezhR1ECBAgEAagTmJ5+ajLI59Bjx7ekGgH4GaV9tJlYctPKlqhY1///Jl2VFNkTz/Cx6JAA6mcyCBHQKCYQeWXQlsEEiXwVFWtxsqeHAJeouSfQj0L+BXhP7nuJcK5+g9vXj/2bNGK7MCbnTiDJvApcCRG6l6St+earmcYz9fE5hieI7ka9vrfU8A1zs3RkaAAAECFwK3svbW+xeHV/WjAK5qOgyGwEACR5bs60wWwes+vW5tbiksgHs9FdVFYGgBGdzl9G9Z5jYUw27C6vIsVdSIAiJnxFlXcwUC0/eRwm6ctgKuYPYMgQABAgTuCWxZ/t5rY+v2sEDd2vq7/QTwOwn/JUCAAIEuBFr5YpJL0F2cboogEFvg9Ldv37x4Grvht+25YJ4Ittdmty9/W0nfaaYEcK+nq7oIRBCI+yfohW6EKdHEqkBD6TvV8Wi1FhsJECBAgEBhgY3L37jpu/wDDCsKYZ8Z+wx4hdQmAgQIECCQSkAAp5LVLgECBAgcFyiy/D0+7C0tCOAtSvYhQIAAgXoF4l58PtW566ryruvVs6MAnim8IECAAIG6BDYuf+sa9ObRCODNVHYkQIAAAQLxBARwPEstESBAgEA8gb6Xv5OTAI53smiJAAECBAhsFhDAm6nsSIAAAQK5BLYvf1PcgZWnSgGcx1kvBAgQINCYwK4boQNqE8ABaA4hQIAAgYQC25e/CQeRvmkBnN5YDwQIECDQu0DAV4EFcO8nhfoIECDQlMAgy99pTgRwUyemwRIgQIBALwICuJeZVAcBAgTaFxhn+TvNlQBu/4RVAQECBAg0KCCAG5w0QyZAgECPAhUuf5N+E0kA93gWq4kAAQIEqhcQwNVPkQESIEBgAIHg5W/wgcVRBXDxKTAAAgQIEBhRQACPOOtqJkCAQFUC7a5izxn3PotDAJ/reU2AAAECBDIJCOBM0LohQIAAgasCfSx/r5a2/qYAXvexlQABAgRqF0ga4em+iSSAaz+xjI8AAQIdCyTNzsrdBHDlE2R4BAgQINCngADuc15VRYAAgfoFRl7+TrMjgOs/RY2QAAECBDoUEMAdTqqSCBAgUL9A3OVv3NaC9XZ9Ffi94G4cSIAAAQIERhA4vxF6V8Su4wjgdR9bCRAgQCC+QCUL1oDCIoaxS9AB/g4hQIAAAQJHBR4dbcDxBAgQIEBgj0C65e/7z57tGUjMfedL0+dL5PUOXIJe97GVAAECBAjcF9ieu3NbLkHPFF4QIECAQHKBdMvf5EOP3YEAji2qPQIECBAgsEFAAG9AsgsBAgQIxBCw/D1XFMDnGl4TIECAQMMCbQW8AG74VDN0AgQINCTQVjpmgBXAGZB1QYAAAQIELgUE8KWInwkQIECAQAYBAZwBWRcECBAg8JDnKRkNXegWwP5fQYAAAQIECggI4ALouiRAgAABAgLYOUCAAAECBAoICOAC6LokQIAAAQIC2DlAgAABApkE3Id1Di2AzzW8JkCAAAECmQQEcCZo3RAgQIAAgXMBAXyu4TUBAgQIEMgkIIAzQeuGAAECBAicCwjgcw2vCRAgQCCtQJ77sNLWEKl1ARwJUjMECBAgQGCPgADeo2VfAgQIECAQSUAAR4LUDAECBAgQ2CMggPdo2ZcAAQIECEQSEMCRIDVDgAABAgT2CAjgPVr2JUCAAAECkQQEcCRIzRAgQIAAgT0CAniPln0JECBAgEAkAQEcCVIzBAgQIEBgj4AA3qNlXwIECBAgEElAAEeC1AwBAgQIENgjIID3aNmXAAECBA4LeBz0iVAAHz6VNECAAAECBPYLCOD9Zo4gQIAAAQKHBQTwYUINECBAgACB/QICeL+ZIwgQIECAwGEBAXyYUAMECBAgQGC/gADeb+YIAgQIEKhYoJW7rAVwxSeRoREgQIBAvwICuN+5VRkBAgTGE2hl+TvNjAAe7/RUMQECBEoLNBST6agEcDpbLRMgQIDATYEUGZyizZsFHN4ggA8TaoAAAQIEggSmvGwrMoOqvHmQAL5JYwMBAgQIZBAYNoYfZcDVBQECBAgQ2CLw/cuXW3a7uk9zi2kr4Kvz6E0CBAgQKCDQXIgWMNIlAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQKBHgf8HhJHrM39VM0gAAAAASUVORK5CYII=", "text/plain": [ "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -272,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "fd651c08-c554-4fb2-9dab-4b44679c500d", "metadata": {}, "outputs": [ @@ -280,7 +331,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "dining table\n" + "person\n" ] }, { @@ -298,7 +349,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "book\n" + "clock\n" ] }, { @@ -316,7 +367,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "potted plant\n" + "refrigerator\n" ] }, { @@ -334,7 +385,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "person\n" + "potted plant\n" ] }, { @@ -352,7 +403,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "chair\n" + "dining table\n" ] }, { @@ -370,7 +421,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "clock\n" + "chair\n" ] }, { @@ -388,7 +439,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "refrigerator\n" + "book\n" ] }, { @@ -406,7 +457,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "microwave\n" + "tv\n" ] }, { @@ -424,7 +475,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "tv\n" + "microwave\n" ] }, { @@ -481,142 +532,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "50f5d932", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.33663366336633666,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.6633663366336634,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.6633663366336634,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.25742574257425743,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.25742574257425743,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.6633663366336634,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.6633663366336634,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", - " {'type': 'AR',\n", + "[{'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -627,8 +550,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -641,8 +564,16 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -654,7 +585,7 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -666,9 +597,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.5,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -680,7 +619,7 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -692,9 +631,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': -1.0,\n", - " 'label': {'key': 'name', 'value': 'bottle'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -705,8 +652,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.35,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -718,9 +665,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -731,8 +686,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.6,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " 'value': 0.5160396039603958,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -744,22 +699,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.6,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'AR',\n", - " 'parameters': {'ious': [0.5,\n", - " 0.55,\n", - " 0.6,\n", - " 0.65,\n", - " 0.7,\n", - " 0.75,\n", - " 0.8,\n", - " 0.85,\n", - " 0.9,\n", - " 0.95]},\n", - " 'value': 0.225,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'AR',\n", + " 'value': 0.5333333333333334,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.5980198019801971,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.8316831683168316,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -783,9 +733,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -797,7 +755,7 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -809,9 +767,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.45,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -822,8 +788,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4999999999999999,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -836,27 +802,29 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", - " {'type': 'mAP',\n", - " 'parameters': {'iou': 0.5, 'label_key': 'name'},\n", - " 'value': 0.4127475247524752},\n", - " {'type': 'mAP',\n", - " 'parameters': {'iou': 0.75, 'label_key': 'name'},\n", - " 'value': 0.40222772277227725},\n", - " {'type': 'mAR',\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", - " 0.7,\n", " 0.65,\n", + " 0.7,\n", " 0.75,\n", " 0.8,\n", " 0.85,\n", " 0.9,\n", - " 0.95],\n", - " 'label_key': 'name'},\n", - " 'value': 0.33906250000000004},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -867,8 +835,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.403960396039604,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -881,8 +857,8 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -894,7 +870,15 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -906,9 +890,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.5049504950495048,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -920,7 +904,15 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -932,9 +924,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -945,8 +937,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.6,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -959,8 +959,8 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -972,7 +972,15 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -984,21 +992,29 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.49801980198019813,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'mAPAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", - " 0.7,\n", " 0.65,\n", + " 0.7,\n", " 0.75,\n", " 0.8,\n", " 0.85,\n", " 0.9,\n", - " 0.95],\n", - " 'label_key': 'name'},\n", - " 'value': 0.34028465346534653},\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1010,9 +1026,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.35346534653465345,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.10099009900990104,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1023,8 +1039,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.597029702970297,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " 'value': 0.3,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.16831683168316838,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.16831683168316838,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1036,9 +1060,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.23168316831683172,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0024752475247524753,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1049,8 +1073,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9009900990099009,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", + " 'value': 0.008333333333333333,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.012376237623762377,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1062,9 +1094,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4544554455445544,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1076,102 +1108,16 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}}]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# bounding box evaluation\n", - "eval_bbox = valor_model_bbox.evaluate_detection(\n", - " valor_dataset,\n", - " filters=Filter(\n", - " labels=(\n", - " Label.key == \"name\"\n", - " )\n", - " )\n", - ")\n", - "eval_bbox.wait_for_completion()\n", - "eval_bbox.metrics" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "e55b1f1e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.33663366336633666,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", - " 'value': 0.33663366336633666,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", " {'type': 'AP',\n", " 'parameters': {'iou': 0.75},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", " {'type': 'AP',\n", - " 'parameters': {'iou': 0.75},\n", + " 'parameters': {'iou': 0.5},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1182,8 +1128,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1196,8 +1142,16 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1208,8 +1162,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1221,9 +1175,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1234,8 +1196,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", + " 'value': 0.016633663366336638,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1247,9 +1209,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", + " 'value': 0.08,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.04158415841584159,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1260,8 +1230,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1273,25 +1243,30 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.1,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'mAP',\n", - " 'parameters': {'iou': 0.5, 'label_key': 'name'},\n", - " 'value': 0.41625412541254125},\n", - " {'type': 'mAR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", - " 0.7,\n", " 0.65,\n", + " 0.7,\n", " 0.75,\n", " 0.8,\n", " 0.85,\n", " 0.9,\n", - " 0.95],\n", - " 'label_key': 'name'},\n", - " 'value': 0.26197916666666665},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 0.95]},\n", + " 'value': 0.0007076117222169536,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1302,8 +1277,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9009900990099009,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", + " 'value': 0.007665505226480835,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0023031876358367547,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1315,9 +1298,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.30297029702970296,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.016831683168316833,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1328,8 +1311,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", + " 'value': 0.03333333333333333,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.16831683168316833,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1342,8 +1333,8 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1354,8 +1345,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.5049504950495048,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1367,9 +1366,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.04488448844884489,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1380,8 +1379,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.5009900990099011,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1393,72 +1400,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.12871287128712872,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'tv'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.6633663366336634,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.504950495049505,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.22442244224422447,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 1.0,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'AP',\n", - " 'parameters': {'iou': 0.5},\n", - " 'value': 0.25742574257425743,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", + " 'value': 0.010396039603960397,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1470,9 +1413,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", + " 'value': 0.02,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.034653465346534656,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1483,8 +1434,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': -1.0,\n", - " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1496,9 +1447,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.3,\n", - " 'label': {'key': 'name', 'value': 'person'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1510,7 +1469,7 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'car'}},\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1523,8 +1482,16 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'book'}},\n", - " {'type': 'AR',\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1535,8 +1502,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.5,\n", - " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1548,9 +1515,17 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.06666666666666667,\n", - " 'label': {'key': 'name', 'value': 'chair'}},\n", - " {'type': 'AR',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1561,8 +1536,8 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.4999999999999999,\n", - " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " 'value': 0.201980198019802,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1574,11 +1549,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.125,\n", - " 'label': {'key': 'name', 'value': 'vase'}},\n", - " {'type': 'mAP',\n", - " 'parameters': {'iou': 0.75, 'label_key': 'name'},\n", - " 'value': 0.26175742574257427},\n", + " 'value': 0.19999999999999998,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.33663366336633666,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.33663366336633666,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1591,8 +1571,8 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'microwave'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1604,7 +1584,15 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1616,9 +1604,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.39900990099009903,\n", - " 'label': {'key': 'name', 'value': 'potted plant'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1629,8 +1617,16 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.403960396039604,\n", - " 'label': {'key': 'name', 'value': 'bed'}},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1643,8 +1639,8 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'truck'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1656,7 +1652,15 @@ " 0.9,\n", " 0.95]},\n", " 'value': 0.0,\n", - " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", @@ -1668,9 +1672,9 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.9,\n", - " 'label': {'key': 'name', 'value': 'bear'}},\n", - " {'type': 'APAveragedOverIOUs',\n", + " 'value': 0.014851485148514854,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AR',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", @@ -1681,13 +1685,3862 @@ " 0.85,\n", " 0.9,\n", " 0.95]},\n", - " 'value': 0.1,\n", - " 'label': {'key': 'name', 'value': 'clock'}},\n", - " {'type': 'mAPAveragedOverIOUs',\n", + " 'value': 0.07142857142857142,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.02970297029702971,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'APAveragedOverIOUs',\n", " 'parameters': {'ious': [0.5,\n", " 0.55,\n", " 0.6,\n", - " 0.7,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.00891089108910891,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.016666666666666666,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.04455445544554455,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.4,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.4,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 1.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.2524752475247525,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.25,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.504950495049505,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.00035360678925035356,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0047619047619047615,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0035360678925035354,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.008415841584158416,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.016666666666666666,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.08415841584158416,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0504950495049505,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.1,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.2524752475247525,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.11782178217821782,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.11666666666666665,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.16831683168316833,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.16831683168316833,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0028052805280528044,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.016666666666666666,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.028052805280528045,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.001386138613861386,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.006666666666666666,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.01386138613861386,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': -1.0,\n", + " 'label': {'key': 'name', 'value': 'snowboard'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': -1.0,\n", + " 'label': {'key': 'name', 'value': 'cow'}},\n", + " {'type': 'mAPAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.7,\n", + " 0.65,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95],\n", + " 'label_key': 'name'},\n", + " 'value': 0.02393846485189652},\n", + " {'type': 'mAR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.7,\n", + " 0.65,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95],\n", + " 'label_key': 'name'},\n", + " 'value': 0.030308184927087367},\n", + " {'type': 'mAP',\n", + " 'parameters': {'iou': 0.75, 'label_key': 'name'},\n", + " 'value': 0.017656765676567644},\n", + " {'type': 'mAP',\n", + " 'parameters': {'iou': 0.5, 'label_key': 'name'},\n", + " 'value': 0.05174272526700986}]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# bounding box evaluation\n", + "eval_bbox = valor_model.evaluate_detection(\n", + " valor_dataset_bbox,\n", + " filters=Filter(\n", + " labels=(\n", + " Label.key == \"name\"\n", + " )\n", + " )\n", + ")\n", + "eval_bbox.wait_for_completion()\n", + "eval_bbox.metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e55b1f1e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'suitcase'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'clock'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bicycle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bottle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'pizza'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'microwave'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skateboard'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'oven'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'surfboard'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0297029702970297,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.02631578947368421,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0594059405940594,\n", + " 'label': {'key': 'name', 'value': 'tie'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.047524752475247525,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.04705882352941177,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0594059405940594,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0594059405940594,\n", + " 'label': {'key': 'name', 'value': 'chair'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sink'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'handbag'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.12871287128712872,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.125,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.25742574257425743,\n", + " 'label': {'key': 'name', 'value': 'mouse'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'apple'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'wine glass'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'couch'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'orange'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dog'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cell phone'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fire hydrant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.01801980198019802,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.016027874564459928,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.019801980198019802,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.019801980198019802,\n", + " 'label': {'key': 'name', 'value': 'person'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'frisbee'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'tennis racket'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'airplane'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'broccoli'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball glove'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'refrigerator'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sheep'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'kite'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.26930693069306927,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.26666666666666666,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.33663366336633666,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.33663366336633666,\n", + " 'label': {'key': 'name', 'value': 'toilet'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'stop sign'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'backpack'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'laptop'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cup'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'dining table'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.17178217821782177,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.16999999999999998,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.2079207920792079,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.2079207920792079,\n", + " 'label': {'key': 'name', 'value': 'tv'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bed'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bear'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'teddy bear'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'vase'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'umbrella'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'zebra'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'train'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sandwich'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'banana'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'donut'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bird'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'car'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'baseball bat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'knife'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'motorcycle'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'traffic light'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'carrot'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'scissors'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'spoon'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'potted plant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cake'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'remote'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'skis'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'cat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bus'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'fork'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'book'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'boat'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'horse'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'sports ball'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'elephant'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.08316831683168316,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.07777777777777779,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.1188118811881188,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.1188118811881188,\n", + " 'label': {'key': 'name', 'value': 'keyboard'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bench'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'truck'}},\n", + " {'type': 'APAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.65,\n", + " 0.7,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95]},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.75},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'AP',\n", + " 'parameters': {'iou': 0.5},\n", + " 'value': 0.0,\n", + " 'label': {'key': 'name', 'value': 'bowl'}},\n", + " {'type': 'mAPAveragedOverIOUs',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.7,\n", + " 0.65,\n", + " 0.75,\n", + " 0.8,\n", + " 0.85,\n", + " 0.9,\n", + " 0.95],\n", + " 'label_key': 'name'},\n", + " 'value': 0.010391914191419141},\n", + " {'type': 'mAR',\n", + " 'parameters': {'ious': [0.5,\n", + " 0.55,\n", + " 0.6,\n", + " 0.7,\n", " 0.65,\n", " 0.75,\n", " 0.8,\n", @@ -1695,18 +5548,24 @@ " 0.9,\n", " 0.95],\n", " 'label_key': 'name'},\n", - " 'value': 0.2616542904290429}]" + " 'value': 0.010122874055722228},\n", + " {'type': 'mAP',\n", + " 'parameters': {'iou': 0.75, 'label_key': 'name'},\n", + " 'value': 0.010313531353135313},\n", + " {'type': 'mAP',\n", + " 'parameters': {'iou': 0.5, 'label_key': 'name'},\n", + " 'value': 0.014713971397139715}]" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# raster evaluation\n", - "eval_raster = valor_model_seg.evaluate_detection(\n", - " valor_dataset,\n", + "eval_raster = valor_model.evaluate_detection(\n", + " valor_dataset_raster,\n", " filters=Filter(\n", " labels=(\n", " Label.key == \"name\"\n", @@ -1728,7 +5587,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "7f4212ea", "metadata": {}, "outputs": [ @@ -1780,29 +5639,29 @@ " \n", " AP\n", " {\"iou\": 0.5}\n", - " name: bear\n", - " 1.000000\n", - " 1.000000\n", + " name: airplane\n", + " 0.000000\n", + " 0.000000\n", " \n", " \n", - " name: bed\n", - " 0.504950\n", - " 0.504950\n", + " name: apple\n", + " 0.168317\n", + " 0.000000\n", " \n", " \n", - " name: book\n", + " name: backpack\n", " 0.000000\n", " 0.000000\n", " \n", " \n", - " name: car\n", + " name: banana\n", " 0.000000\n", " 0.000000\n", " \n", " \n", - " name: chair\n", - " 0.504950\n", - " 0.224422\n", + " name: baseball bat\n", + " 0.000000\n", + " 0.000000\n", " \n", " \n", " ...\n", @@ -1814,77 +5673,77 @@ " \n", " AR\n", " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", - " name: vase\n", - " 0.225000\n", - " 0.125000\n", + " name: zebra\n", + " 0.000000\n", + " 0.000000\n", " \n", " \n", " mAP\n", " {\"iou\": 0.5, \"label_key\": \"name\"}\n", " n/a\n", - " 0.412748\n", - " 0.416254\n", + " 0.051743\n", + " 0.014714\n", " \n", " \n", " {\"iou\": 0.75, \"label_key\": \"name\"}\n", " n/a\n", - " 0.402228\n", - " 0.261757\n", + " 0.017657\n", + " 0.010314\n", " \n", " \n", " mAPAveragedOverIOUs\n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}\n", " n/a\n", - " 0.340285\n", - " 0.261654\n", + " 0.023938\n", + " 0.010392\n", " \n", " \n", " mAR\n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}\n", " n/a\n", - " 0.339063\n", - " 0.261979\n", + " 0.030308\n", + " 0.010123\n", " \n", " \n", "\n", - "

69 rows × 2 columns

\n", + "

294 rows × 2 columns

\n", "" ], "text/plain": [ - " value \\\n", - "annotation type bbox \n", - "type parameters label \n", - "AP {\"iou\": 0.5} name: bear 1.000000 \n", - " name: bed 0.504950 \n", - " name: book 0.000000 \n", - " name: car 0.000000 \n", - " name: chair 0.504950 \n", - "... ... \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: vase 0.225000 \n", - "mAP {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.412748 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.402228 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.340285 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.339063 \n", + " value \\\n", + "annotation type bbox \n", + "type parameters label \n", + "AP {\"iou\": 0.5} name: airplane 0.000000 \n", + " name: apple 0.168317 \n", + " name: backpack 0.000000 \n", + " name: banana 0.000000 \n", + " name: baseball bat 0.000000 \n", + "... ... \n", + "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: zebra 0.000000 \n", + "mAP {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.051743 \n", + " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.017657 \n", + "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.023938 \n", + "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.030308 \n", "\n", - " \n", - "annotation type raster \n", - "type parameters label \n", - "AP {\"iou\": 0.5} name: bear 1.000000 \n", - " name: bed 0.504950 \n", - " name: book 0.000000 \n", - " name: car 0.000000 \n", - " name: chair 0.224422 \n", - "... ... \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: vase 0.125000 \n", - "mAP {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.416254 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.261757 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.261654 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.261979 \n", + " \n", + "annotation type raster \n", + "type parameters label \n", + "AP {\"iou\": 0.5} name: airplane 0.000000 \n", + " name: apple 0.000000 \n", + " name: backpack 0.000000 \n", + " name: banana 0.000000 \n", + " name: baseball bat 0.000000 \n", + "... ... \n", + "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: zebra 0.000000 \n", + "mAP {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.014714 \n", + " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.010314 \n", + "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.010392 \n", + "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.010123 \n", "\n", - "[69 rows x 2 columns]" + "[294 rows x 2 columns]" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -1909,7 +5768,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "fc78dd1b", "metadata": {}, "outputs": [], @@ -1929,7 +5788,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "c1453301", "metadata": {}, "outputs": [ @@ -1939,15 +5798,15 @@ "" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# bounding box evaluation\n", - "eval_bbox_small = valor_model_bbox.evaluate_detection(\n", - " valor_dataset,\n", + "eval_bbox_small = valor_model.evaluate_detection(\n", + " valor_dataset_bbox,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -1960,7 +5819,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "ef904d0b", "metadata": {}, "outputs": [ @@ -1970,15 +5829,15 @@ "" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# raster evaluation\n", - "eval_raster_small = valor_model_seg.evaluate_detection(\n", - " valor_dataset,\n", + "eval_raster_small = valor_model.evaluate_detection(\n", + " valor_dataset_raster,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -1991,7 +5850,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "cb3dc40d", "metadata": {}, "outputs": [ @@ -2044,28 +5903,28 @@ " AP\n", " {\"iou\": 0.5}\n", " iscrowd: 0\n", - " 0.00000\n", - " 0.000000\n", + " 0.0\n", + " 0.0\n", " \n", " \n", - " name: book\n", - " 0.00000\n", - " 0.000000\n", + " iscrowd: 1\n", + " 0.0\n", + " 0.0\n", " \n", " \n", - " name: car\n", - " 0.00000\n", - " 0.000000\n", + " name: airplane\n", + " 0.0\n", + " 0.0\n", " \n", " \n", - " name: chair\n", - " 0.50495\n", - " 0.224422\n", + " name: backpack\n", + " 0.0\n", + " 0.0\n", " \n", " \n", - " name: clock\n", - " 1.00000\n", - " 1.000000\n", + " name: banana\n", + " 0.0\n", + " 0.0\n", " \n", " \n", " ...\n", @@ -2075,74 +5934,72 @@ " \n", " \n", " {\"iou\": 0.75}\n", - " name: bed\n", + " name: orange\n", " NaN\n", - " 0.000000\n", + " 0.0\n", " \n", " \n", - " APAveragedOverIOUs\n", - " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", - " iscrowd: 1\n", + " APAveragedOverIOUs\n", + " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", + " name: apple\n", " NaN\n", - " 0.000000\n", + " 0.0\n", " \n", " \n", " name: bed\n", " NaN\n", - " 0.000000\n", + " 0.0\n", " \n", " \n", - " AR\n", - " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", - " iscrowd: 1\n", + " name: couch\n", " NaN\n", - " 0.000000\n", + " 0.0\n", " \n", " \n", - " name: bed\n", + " name: orange\n", " NaN\n", - " 0.000000\n", + " 0.0\n", " \n", " \n", "\n", - "

105 rows × 2 columns

\n", + "

332 rows × 2 columns

\n", "" ], "text/plain": [ " value \\\n", "annotation type bbox \n", "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.00000 \n", - " name: book 0.00000 \n", - " name: car 0.00000 \n", - " name: chair 0.50495 \n", - " name: clock 1.00000 \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", + " iscrowd: 1 0.0 \n", + " name: airplane 0.0 \n", + " name: backpack 0.0 \n", + " name: banana 0.0 \n", "... ... \n", - " {\"iou\": 0.75} name: bed NaN \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 1 NaN \n", - " name: bed NaN \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 1 NaN \n", + " {\"iou\": 0.75} name: orange NaN \n", + "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: apple NaN \n", " name: bed NaN \n", + " name: couch NaN \n", + " name: orange NaN \n", "\n", " \n", "annotation type raster \n", "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.000000 \n", - " name: book 0.000000 \n", - " name: car 0.000000 \n", - " name: chair 0.224422 \n", - " name: clock 1.000000 \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", + " iscrowd: 1 0.0 \n", + " name: airplane 0.0 \n", + " name: backpack 0.0 \n", + " name: banana 0.0 \n", "... ... \n", - " {\"iou\": 0.75} name: bed 0.000000 \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 1 0.000000 \n", - " name: bed 0.000000 \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 1 0.000000 \n", - " name: bed 0.000000 \n", + " {\"iou\": 0.75} name: orange 0.0 \n", + "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: apple 0.0 \n", + " name: bed 0.0 \n", + " name: couch 0.0 \n", + " name: orange 0.0 \n", "\n", - "[105 rows x 2 columns]" + "[332 rows x 2 columns]" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -2164,7 +6021,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "id": "472fa53b", "metadata": {}, "outputs": [ @@ -2174,15 +6031,15 @@ "" ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# bounding box evaluation\n", - "eval_bbox_mid = valor_model_bbox.evaluate_detection(\n", - " valor_dataset,\n", + "eval_bbox_mid = valor_model.evaluate_detection(\n", + " valor_dataset_bbox,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -2196,7 +6053,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "id": "28aecb68", "metadata": {}, "outputs": [ @@ -2206,15 +6063,15 @@ "" ] }, - "execution_count": 16, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# raster evaluation\n", - "eval_raster_mid = valor_model_seg.evaluate_detection(\n", - " valor_dataset,\n", + "eval_raster_mid = valor_model.evaluate_detection(\n", + " valor_dataset_raster,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -2228,7 +6085,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "id": "1834644a", "metadata": {}, "outputs": [ @@ -2256,349 +6113,130 @@ "\n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", " \n", "
value
annotation typebboxraster
typeparameterslabel
AP{\"iou\": 0.5}iscrowd: 00.00.000000
iscrowd: 10.0NaN
name: bed1.01.000000
name: book0.0NaN
supercategory: furniture0.00.000000
supercategory: indoor0.00.000000
{\"iou\": 0.75}iscrowd: 00.00.000000
iscrowd: 10.0NaN
name: bed1.01.000000
name: book0.0NaN
supercategory: furniture0.00.000000
supercategory: indoor0.00.000000
APAveragedOverIOUs{\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}iscrowd: 00.00.000000
iscrowd: 10.0NaN
name: bed0.80.800000
name: book0.0NaNvalue
supercategory: furniture0.00.000000annotation typebboxraster
supercategory: indoor0.00.000000typeparameterslabel
AR{\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}AP{\"iou\": 0.5}iscrowd: 00.00.000000
iscrowd: 10.0NaN
name: bed0.80.800000
name: book0.0NaN
name: teddy bear-1.00.466667
supercategory: furniture0.0iscrowd: 10.000000
supercategory: indoor0.00.000000
mAP{\"iou\": 0.5, \"label_key\": \"iscrowd\"}n/aname: apple0.1683170.00.000000
{\"iou\": 0.5, \"label_key\": \"name\"}n/a0.51.000000
{\"iou\": 0.5, \"label_key\": \"supercategory\"}n/a0.0name: banana0.000000
{\"iou\": 0.75, \"label_key\": \"iscrowd\"}n/a0.00.000000
{\"iou\": 0.75, \"label_key\": \"name\"}n/a0.50.668317
{\"iou\": 0.75, \"label_key\": \"supercategory\"}n/a0.0name: bed0.000000
mAPAveragedOverIOUs{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"iscrowd\"}n/a0.00.000000
{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}n/a0.40.628713
{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"supercategory\"}n/a0.00.000000...............
mAR{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"iscrowd\"}n/aAPAveragedOverIOUs{\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}name: zebraNaN0.00.000000
{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}n/a0.40.633333
{\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"supercategory\"}n/aAR{\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}name: airplaneNaN0.00.000000
AP{\"iou\": 0.5}name: teddy bearname: catNaN1.0000000.0
{\"iou\": 0.75}name: teddy bearname: trainNaN0.3366340.0
APAveragedOverIOUs{\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}name: teddy bearname: zebraNaN0.4574260.0
\n", + "

208 rows × 2 columns

\n", "" ], "text/plain": [ - " value \\\n", - "annotation type bbox \n", - "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", - " iscrowd: 1 0.0 \n", - " name: bed 1.0 \n", - " name: book 0.0 \n", - " supercategory: furniture 0.0 \n", - " supercategory: indoor 0.0 \n", - " {\"iou\": 0.75} iscrowd: 0 0.0 \n", - " iscrowd: 1 0.0 \n", - " name: bed 1.0 \n", - " name: book 0.0 \n", - " supercategory: furniture 0.0 \n", - " supercategory: indoor 0.0 \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.0 \n", - " iscrowd: 1 0.0 \n", - " name: bed 0.8 \n", - " name: book 0.0 \n", - " supercategory: furniture 0.0 \n", - " supercategory: indoor 0.0 \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.0 \n", - " iscrowd: 1 0.0 \n", - " name: bed 0.8 \n", - " name: book 0.0 \n", - " name: teddy bear -1.0 \n", - " supercategory: furniture 0.0 \n", - " supercategory: indoor 0.0 \n", - "mAP {\"iou\": 0.5, \"label_key\": \"iscrowd\"} n/a 0.0 \n", - " {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.5 \n", - " {\"iou\": 0.5, \"label_key\": \"supercategory\"} n/a 0.0 \n", - " {\"iou\": 0.75, \"label_key\": \"iscrowd\"} n/a 0.0 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.5 \n", - " {\"iou\": 0.75, \"label_key\": \"supercategory\"} n/a 0.0 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.4 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.4 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - "AP {\"iou\": 0.5} name: teddy bear NaN \n", - " {\"iou\": 0.75} name: teddy bear NaN \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: teddy bear NaN \n", + " value \\\n", + "annotation type bbox \n", + "type parameters label \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.000000 \n", + " iscrowd: 1 0.000000 \n", + " name: apple 0.168317 \n", + " name: banana 0.000000 \n", + " name: bed 0.000000 \n", + "... ... \n", + "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: zebra NaN \n", + "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: airplane NaN \n", + " name: cat NaN \n", + " name: train NaN \n", + " name: zebra NaN \n", + "\n", + " \n", + "annotation type raster \n", + "type parameters label \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", + " iscrowd: 1 0.0 \n", + " name: apple 0.0 \n", + " name: banana 0.0 \n", + " name: bed 0.0 \n", + "... ... \n", + "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: zebra 0.0 \n", + "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: airplane 0.0 \n", + " name: cat 0.0 \n", + " name: train 0.0 \n", + " name: zebra 0.0 \n", "\n", - " \n", - "annotation type raster \n", - "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.000000 \n", - " iscrowd: 1 NaN \n", - " name: bed 1.000000 \n", - " name: book NaN \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - " {\"iou\": 0.75} iscrowd: 0 0.000000 \n", - " iscrowd: 1 NaN \n", - " name: bed 1.000000 \n", - " name: book NaN \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.000000 \n", - " iscrowd: 1 NaN \n", - " name: bed 0.800000 \n", - " name: book NaN \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.000000 \n", - " iscrowd: 1 NaN \n", - " name: bed 0.800000 \n", - " name: book NaN \n", - " name: teddy bear 0.466667 \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "mAP {\"iou\": 0.5, \"label_key\": \"iscrowd\"} n/a 0.000000 \n", - " {\"iou\": 0.5, \"label_key\": \"name\"} n/a 1.000000 \n", - " {\"iou\": 0.5, \"label_key\": \"supercategory\"} n/a 0.000000 \n", - " {\"iou\": 0.75, \"label_key\": \"iscrowd\"} n/a 0.000000 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.668317 \n", - " {\"iou\": 0.75, \"label_key\": \"supercategory\"} n/a 0.000000 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.628713 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.633333 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - "AP {\"iou\": 0.5} name: teddy bear 1.000000 \n", - " {\"iou\": 0.75} name: teddy bear 0.336634 \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... name: teddy bear 0.457426 " + "[208 rows x 2 columns]" ] }, - "execution_count": 17, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -2620,7 +6258,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "id": "63247d38", "metadata": {}, "outputs": [ @@ -2630,15 +6268,15 @@ "" ] }, - "execution_count": 18, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# bounding box evaluation\n", - "eval_bbox_large = valor_model_bbox.evaluate_detection(\n", - " valor_dataset,\n", + "eval_bbox_large = valor_model.evaluate_detection(\n", + " valor_dataset_bbox,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -2651,7 +6289,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "818f8147", "metadata": {}, "outputs": [ @@ -2661,15 +6299,15 @@ "" ] }, - "execution_count": 19, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# raster evaluation\n", - "eval_raster_large = valor_model_seg.evaluate_detection(\n", - " valor_dataset,\n", + "eval_raster_large = valor_model.evaluate_detection(\n", + " valor_dataset_raster,\n", " filters=Filter(\n", " annotations=And(\n", " Label.key == \"name\",\n", @@ -2682,7 +6320,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "id": "f7e20df1", "metadata": {}, "outputs": [ @@ -2732,201 +6370,45 @@ " \n", " \n", " \n", - " AP\n", - " {\"iou\": 0.5}\n", + " AP\n", + " {\"iou\": 0.5}\n", " iscrowd: 0\n", " 0.000000\n", " 0.0\n", " \n", " \n", - " name: bear\n", - " 1.000000\n", - " 1.0\n", - " \n", - " \n", - " name: bed\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " name: teddy bear\n", - " 0.663366\n", - " NaN\n", - " \n", - " \n", - " supercategory: animal\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " supercategory: furniture\n", + " iscrowd: 1\n", " 0.000000\n", " NaN\n", " \n", " \n", - " supercategory: indoor\n", + " name: airplane\n", " 0.000000\n", " NaN\n", " \n", " \n", - " {\"iou\": 0.75}\n", - " iscrowd: 0\n", + " name: banana\n", " 0.000000\n", " 0.0\n", " \n", " \n", " name: bear\n", " 1.000000\n", - " 1.0\n", - " \n", - " \n", - " name: bed\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " name: teddy bear\n", - " 0.663366\n", - " NaN\n", - " \n", - " \n", - " supercategory: animal\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " supercategory: furniture\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " supercategory: indoor\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " APAveragedOverIOUs\n", - " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", - " iscrowd: 0\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " name: bear\n", - " 0.900000\n", - " 0.9\n", - " \n", - " \n", - " name: bed\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " name: teddy bear\n", - " 0.597030\n", - " NaN\n", - " \n", - " \n", - " supercategory: animal\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " supercategory: furniture\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " supercategory: indoor\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " AR\n", - " {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]}\n", - " iscrowd: 0\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " name: bear\n", - " 0.900000\n", - " 0.9\n", - " \n", - " \n", - " name: bed\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " name: teddy bear\n", - " 0.600000\n", - " -1.0\n", - " \n", - " \n", - " supercategory: animal\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " supercategory: furniture\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " supercategory: indoor\n", - " 0.000000\n", - " NaN\n", - " \n", - " \n", - " mAP\n", - " {\"iou\": 0.5, \"label_key\": \"iscrowd\"}\n", - " n/a\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " {\"iou\": 0.5, \"label_key\": \"name\"}\n", - " n/a\n", - " 0.554455\n", - " 1.0\n", - " \n", - " \n", - " {\"iou\": 0.5, \"label_key\": \"supercategory\"}\n", - " n/a\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " {\"iou\": 0.75, \"label_key\": \"iscrowd\"}\n", - " n/a\n", - " 0.000000\n", - " 0.0\n", - " \n", - " \n", - " {\"iou\": 0.75, \"label_key\": \"name\"}\n", - " n/a\n", - " 0.554455\n", - " 1.0\n", - " \n", - " \n", - " {\"iou\": 0.75, \"label_key\": \"supercategory\"}\n", - " n/a\n", - " 0.000000\n", " 0.0\n", " \n", " \n", - " mAPAveragedOverIOUs\n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"iscrowd\"}\n", - " n/a\n", - " 0.000000\n", - " 0.0\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", " \n", " \n", + " mAPAveragedOverIOUs\n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}\n", " n/a\n", - " 0.499010\n", - " 0.9\n", + " 0.109542\n", + " 0.0\n", " \n", " \n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"supercategory\"}\n", @@ -2944,8 +6426,8 @@ " \n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"name\"}\n", " n/a\n", - " 0.500000\n", - " 0.9\n", + " 0.117353\n", + " 0.0\n", " \n", " \n", " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8, 0.85, 0.9, 0.95], \"label_key\": \"supercategory\"}\n", @@ -2955,99 +6437,44 @@ " \n", " \n", "\n", + "

128 rows × 2 columns

\n", "" ], "text/plain": [ - " value \\\n", - "annotation type bbox \n", - "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.000000 \n", - " name: bear 1.000000 \n", - " name: bed 0.000000 \n", - " name: teddy bear 0.663366 \n", - " supercategory: animal 0.000000 \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - " {\"iou\": 0.75} iscrowd: 0 0.000000 \n", - " name: bear 1.000000 \n", - " name: bed 0.000000 \n", - " name: teddy bear 0.663366 \n", - " supercategory: animal 0.000000 \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.000000 \n", - " name: bear 0.900000 \n", - " name: bed 0.000000 \n", - " name: teddy bear 0.597030 \n", - " supercategory: animal 0.000000 \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.000000 \n", - " name: bear 0.900000 \n", - " name: bed 0.000000 \n", - " name: teddy bear 0.600000 \n", - " supercategory: animal 0.000000 \n", - " supercategory: furniture 0.000000 \n", - " supercategory: indoor 0.000000 \n", - "mAP {\"iou\": 0.5, \"label_key\": \"iscrowd\"} n/a 0.000000 \n", - " {\"iou\": 0.5, \"label_key\": \"name\"} n/a 0.554455 \n", - " {\"iou\": 0.5, \"label_key\": \"supercategory\"} n/a 0.000000 \n", - " {\"iou\": 0.75, \"label_key\": \"iscrowd\"} n/a 0.000000 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 0.554455 \n", - " {\"iou\": 0.75, \"label_key\": \"supercategory\"} n/a 0.000000 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.499010 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.500000 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", + " value \\\n", + "annotation type bbox \n", + "type parameters label \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.000000 \n", + " iscrowd: 1 0.000000 \n", + " name: airplane 0.000000 \n", + " name: banana 0.000000 \n", + " name: bear 1.000000 \n", + "... ... \n", + "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.109542 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", + "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.117353 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.000000 \n", + "\n", + " \n", + "annotation type raster \n", + "type parameters label \n", + "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", + " iscrowd: 1 NaN \n", + " name: airplane NaN \n", + " name: banana 0.0 \n", + " name: bear 0.0 \n", + "... ... \n", + "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", + "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", + " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", "\n", - " \n", - "annotation type raster \n", - "type parameters label \n", - "AP {\"iou\": 0.5} iscrowd: 0 0.0 \n", - " name: bear 1.0 \n", - " name: bed NaN \n", - " name: teddy bear NaN \n", - " supercategory: animal 0.0 \n", - " supercategory: furniture NaN \n", - " supercategory: indoor NaN \n", - " {\"iou\": 0.75} iscrowd: 0 0.0 \n", - " name: bear 1.0 \n", - " name: bed NaN \n", - " name: teddy bear NaN \n", - " supercategory: animal 0.0 \n", - " supercategory: furniture NaN \n", - " supercategory: indoor NaN \n", - "APAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.0 \n", - " name: bear 0.9 \n", - " name: bed NaN \n", - " name: teddy bear NaN \n", - " supercategory: animal 0.0 \n", - " supercategory: furniture NaN \n", - " supercategory: indoor NaN \n", - "AR {\"ious\": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8,... iscrowd: 0 0.0 \n", - " name: bear 0.9 \n", - " name: bed NaN \n", - " name: teddy bear -1.0 \n", - " supercategory: animal 0.0 \n", - " supercategory: furniture NaN \n", - " supercategory: indoor NaN \n", - "mAP {\"iou\": 0.5, \"label_key\": \"iscrowd\"} n/a 0.0 \n", - " {\"iou\": 0.5, \"label_key\": \"name\"} n/a 1.0 \n", - " {\"iou\": 0.5, \"label_key\": \"supercategory\"} n/a 0.0 \n", - " {\"iou\": 0.75, \"label_key\": \"iscrowd\"} n/a 0.0 \n", - " {\"iou\": 0.75, \"label_key\": \"name\"} n/a 1.0 \n", - " {\"iou\": 0.75, \"label_key\": \"supercategory\"} n/a 0.0 \n", - "mAPAveragedOverIOUs {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.9 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - "mAR {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.9 \n", - " {\"ious\": [0.5, 0.55, 0.6, 0.7, 0.65, 0.75, 0.8,... n/a 0.0 " + "[128 rows x 2 columns]" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -3075,7 +6502,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/examples/object-detection/integrations/coco_integration.py b/examples/object-detection/integrations/coco_integration.py index b8afe45ee..3c2ab3aca 100644 --- a/examples/object-detection/integrations/coco_integration.py +++ b/examples/object-detection/integrations/coco_integration.py @@ -4,17 +4,59 @@ import zipfile from io import BytesIO from pathlib import Path -from typing import Dict, List, Union +from typing import Dict, Generator, List, Union import numpy as np import PIL.Image import requests +from shapely import geometry, ops +from skimage import measure from tqdm import tqdm -from valor import Annotation, Client, Dataset, Datum, GroundTruth, Label -from valor.enums import TaskType +from valor import Annotation, Datum, GroundTruth, Label +from valor.enums import AnnotationType, TaskType from valor.metatypes import ImageMetadata -from valor.schemas import Raster +from valor.schemas import Box, MultiPolygon, Polygon, Raster + + +def download_image(datum: Datum) -> PIL.Image.Image: + """ + Download image using Datum. + """ + url = datum.metadata["coco_url"] + if not isinstance(url, str): + raise TypeError("datum.metadata['coco_url'] is not type 'str'.") + img_data = BytesIO(requests.get(url).content) + return PIL.Image.open(img_data) + + +def download_data_if_not_exists( + filename: str, + filepath: Path, + url: str, +): + """Download the data from a public bucket if it doesn't exist locally.""" + + if not os.path.exists(filepath): + response = requests.get(url, stream=True) + if response.status_code == 200: + total_size = int(response.headers.get("content-length", 0)) + with open(filepath, "wb") as f: + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=filename, + ) as pbar: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + pbar.update(1024) + else: + raise RuntimeError(response) + else: + print(f"{filename} already exists locally.") def download_coco_panoptic( @@ -22,7 +64,19 @@ def download_coco_panoptic( coco_url: str = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip", ) -> dict: """ - Download and return COCO panoptic dataset. + Download the COCO panoptic dataset. + + Parameters + ---------- + destination: Path + The filepath where the dataset will be stored. + coco_url: str + The url where the coco dataset is stored. + + Returns + ------- + dict + A dictionary containing the coco dataset. """ # append the location of the annotations within the destination folder @@ -79,20 +133,117 @@ def download_coco_panoptic( return panoptic_val2017 -def download_image(datum: Datum) -> PIL.Image.Image: +def convert_bitmask_to_bbox(bitmask: np.ndarray) -> Box: """ - Download image using Datum. + Converts a bitmask to a Valor Box schema. + + Parameters + ---------- + bitmask: np.ndarray + The bitmask to convert. + + Returns + ------- + valor.schemas.Box """ - url = datum.metadata["coco_url"] - if not isinstance(url, str): - raise TypeError("datum.metadata['coco_url'] is not type 'str'.") - img_data = BytesIO(requests.get(url).content) - return PIL.Image.open(img_data) + bitmask = np.array(bitmask, dtype=bool) + true_indices = np.argwhere(bitmask) + if true_indices.size == 0: + raise RuntimeError + + xmin, ymin = true_indices.min(axis=0) + xmax, ymax = true_indices.max(axis=0) + + return Box.from_extrema( + xmin=float(xmin), + xmax=float(xmax), + ymin=float(ymin), + ymax=float(ymax), + ) + + +def convert_bitmask_to_multipolygon_raster(bitmask: np.ndarray) -> Raster: + """ + Converts a bitmask to a Valor Raster schema. + + Parameters + ---------- + bitmask: np.ndarray + The bitmask to convert. + + Returns + ------- + valor.schemas.Raster + """ + bitmask = np.array(bitmask, dtype=bool) + labeled_array, num_features = measure.label( + bitmask, background=0, return_num=True + ) # type: ignore - skimage + polygons = [] + for region_index in range(1, num_features + 1): + contours = measure.find_contours(labeled_array == region_index, 0.5) + for contour in contours: + if len(contour) >= 3: + polygon = geometry.Polygon(contour) + if polygon.is_valid: + polygons.append(polygon) + mp = geometry.MultiPolygon(polygons).simplify(tolerance=0.6) + values = [] + if isinstance(mp, geometry.MultiPolygon): + for polygon in mp.geoms: + boundary = list(polygon.exterior.coords) + holes = [list(interior.coords) for interior in polygon.interiors] + values.append([boundary, *holes]) + else: + boundary = list(mp.exterior.coords) + holes = [list(interior.coords) for interior in mp.interiors] + values = [[boundary, *holes]] + height, width = bitmask.shape + return Raster.from_geometry( + MultiPolygon(values), height=height, width=width + ) + + +def convert_bitmask_to_polygon(bitmask: np.ndarray) -> Polygon: + """ + Converts a bitmask to a Valor Polygon schema. + + Parameters + ---------- + bitmask: np.ndarray + The bitmask to convert. + + Returns + ------- + valor.schemas.Polygon + """ + bitmask = np.array(bitmask, dtype=bool) + labeled_array, num_features = measure.label( + bitmask, + background=0, + return_num=True, + ) # type: ignore - skimage + polygons = [] + for region_index in range(1, num_features + 1): + contours = measure.find_contours(labeled_array == region_index, 0.5) + for contour in contours: + if len(contour) >= 3: + polygon = geometry.Polygon(contour) + if polygon.is_valid: + polygons.append(polygon) + polygon = ops.unary_union( + geometry.MultiPolygon(polygons).simplify(tolerance=0.6) + ) + if not isinstance(polygon, geometry.Polygon): + return None + boundary = list(polygon.exterior.coords) + holes = [list(interior.coords) for interior in polygon.interiors] + return Polygon([boundary, *holes]) def _parse_image_to_datum(image: dict) -> Datum: """ - Parse COCO image to Valor Datum + Parse COCO image to Valor Datum. """ image = image.copy() uid = str(image.pop("id")) @@ -111,7 +262,7 @@ def _parse_categories( categories: List[dict], ) -> Dict[int, Union[bool, Dict[str, str]]]: """ - Parse COCO categories into `valor.enums.TaskType` and `valor.Label` + Parse COCO categories into `valor.enums.TaskType` and `valor.Label`. """ return { category["id"]: { @@ -133,11 +284,107 @@ def _create_masks(filename: str) -> np.ndarray: return mask[:, :, 0] + 256 * mask[:, :, 1] + (256**2) * mask[:, :, 2] -def create_annotations_from_instance_segmentations( +def create_bounding_boxes( + image: dict, + category_id_to_labels_and_task: Dict[int, Union[TaskType, Dict[str, str]]], + mask_ids, +) -> List[Annotation]: + """ + Create bounding box annotations from COCO annotations. + """ + return [ + Annotation( + labels=[ + Label( + key="supercategory", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "supercategory" + ] # type: ignore - dict typing + ), + ), + Label( + key="name", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "name" + ] # type: ignore - dict typing + ), + ), + Label(key="iscrowd", value=str(segmentation["iscrowd"])), + ], + bounding_box=convert_bitmask_to_bbox( + mask_ids == segmentation["id"] + ), + is_instance=True, + ) + for segmentation in image["segments_info"] + if category_id_to_labels_and_task[segmentation["category_id"]][ + "is_instance" + ] # type: ignore - dict typing + is True + and convert_bitmask_to_bbox(mask_ids == segmentation["id"]) is not None + ] + + +def create_bounding_polygons( + image: dict, + category_id_to_labels_and_task: Dict[int, Union[TaskType, Dict[str, str]]], + mask_ids, +) -> List[Annotation]: + """ + Create bounding polygon annotations from COCO annotations. + """ + return [ + Annotation( + labels=[ + Label( + key="supercategory", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "supercategory" + ] # type: ignore - dict typing + ), + ), + Label( + key="name", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "name" + ] # type: ignore - dict typing + ), + ), + Label(key="iscrowd", value=str(segmentation["iscrowd"])), + ], + polygon=convert_bitmask_to_polygon(mask_ids == segmentation["id"]), + is_instance=True, + ) + for segmentation in image["segments_info"] + if category_id_to_labels_and_task[segmentation["category_id"]][ + "is_instance" + ] # type: ignore - dict typing + is True + and convert_bitmask_to_polygon(mask_ids == segmentation["id"]) + is not None + ] + + +def create_raster_from_bitmask( image: dict, category_id_to_labels_and_task: Dict[int, Union[TaskType, Dict[str, str]]], mask_ids, ) -> List[Annotation]: + """ + Create raster annotations from COCO annotations. + """ return [ Annotation( labels=[ @@ -174,11 +421,65 @@ def create_annotations_from_instance_segmentations( ] -def create_annotations_from_semantic_segmentations( +def create_raster_from_multipolygon( image: dict, category_id_to_labels_and_task: Dict[int, Union[TaskType, Dict[str, str]]], mask_ids, +) -> List[Annotation]: + """ + Create multipolygon annotations from COCO annotations. + """ + return [ + Annotation( + labels=[ + Label( + key="supercategory", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "supercategory" + ] # type: ignore - dict typing + ), + ), + Label( + key="name", + value=str( + category_id_to_labels_and_task[ + segmentation["category_id"] + ]["labels"][ + "name" + ] # type: ignore - dict typing + ), + ), + Label(key="iscrowd", value=str(segmentation["iscrowd"])), + ], + raster=convert_bitmask_to_multipolygon_raster( + mask_ids == segmentation["id"] + ), + is_instance=True, + ) + for segmentation in image["segments_info"] + if category_id_to_labels_and_task[segmentation["category_id"]][ + "is_instance" + ] # type: ignore - dict typing + is True + ] + + +def create_semantic_segmentations( + image: dict, + category_id_to_labels_and_task: Dict[int, Union[TaskType, Dict[str, str]]], + mask_ids, + dtype: AnnotationType = AnnotationType.RASTER, ): + """ + Create semantic annotations from COCO annotations. + """ + + if dtype not in [AnnotationType.MULTIPOLYGON, AnnotationType.RASTER]: + raise ValueError(dtype) + # combine semantic segmentations semantic_masks = { "supercategory": {}, @@ -218,7 +519,13 @@ def create_annotations_from_semantic_segmentations( return [ Annotation( labels=[Label(key=key, value=str(value))], - raster=Raster.from_numpy(semantic_masks[key][value]), + raster=( + Raster.from_numpy(semantic_masks[key][value]) + if dtype == AnnotationType.RASTER + else convert_bitmask_to_multipolygon_raster( + semantic_masks[key][value] + ) + ), is_instance=False, ) for key in semantic_masks @@ -226,115 +533,301 @@ def create_annotations_from_semantic_segmentations( ] -def _create_groundtruths_from_coco_panoptic( - data: dict, - masks_path: Path, -) -> List[GroundTruth]: - # extract labels from categories - category_id_to_labels_and_task = _parse_categories(data["categories"]) - # create datums - image_id_to_datum = { - image["id"]: _parse_image_to_datum(image) for image in data["images"] - } +def create_instance_groundtruths_file( + dtype: AnnotationType, + filename: str, + path: Path, + destination: Path, + coco_url: str, + limit: int, +): + if dtype not in [ + AnnotationType.BOX, + AnnotationType.POLYGON, + AnnotationType.MULTIPOLYGON, + AnnotationType.RASTER, + ]: + raise ValueError(dtype) - # create groundtruths - groundtruths = [] - for image in tqdm(data["annotations"], "Formatting"): - # exract masks from annotations - mask_ids = _create_masks(masks_path / image["file_name"]) - - # create instance segmentations - instance_annotations = create_annotations_from_instance_segmentations( - image, - category_id_to_labels_and_task, - mask_ids, - ) + # download and unzip coco dataset + coco_path = Path(path) / Path(destination) + data = download_coco_panoptic( + destination=coco_path, + coco_url=coco_url, + ) - # create semantic segmentations - semantic_annotations = create_annotations_from_semantic_segmentations( - image, - category_id_to_labels_and_task, - mask_ids, - ) + # path of mask locations + masks_path = coco_path / Path("annotations/panoptic_val2017/") - # create groundTruth - groundtruths.append( - GroundTruth( - datum=image_id_to_datum[image["image_id"]], - annotations=instance_annotations + semantic_annotations, - ) - ) + # slice if limited + if limit > 0: + data["annotations"] = data["annotations"][:limit] - return groundtruths + # get filepath + filepath = Path(path) / Path(filename) + # get creator function + functions = { + AnnotationType.BOX: create_bounding_boxes, + AnnotationType.POLYGON: create_bounding_polygons, + AnnotationType.MULTIPOLYGON: create_raster_from_multipolygon, + AnnotationType.RASTER: create_raster_from_bitmask, + } + create = functions[dtype] -def create_dataset_from_coco_panoptic( - name: str = "coco2017-panoptic-semseg", - destination: str = "./coco", - coco_url: str = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip", - limit: int = 0, - delete_if_exists: bool = False, -) -> Dataset: - """ - Creates Dataset and associated GroundTruths. + with open(filepath, mode="wb") as f: - Parameters - ---------- - name : str - Desired dataset name. - destination : str - Desired output path for dataset annotations. - coco_url : str - URL to the COCO dataset. - annotations_zipfile : Path - Local path to annotations zipfile. - masks_path : Path - Local path to unzipped annotations. - limit : int, default=0 - Limits the number of datums. Default to 0 for no action. - delete_if_exists : bool, default=False - Reset the Valor dataset before attempting creation. + # extract labels from categories + category_id_to_labels_and_task = _parse_categories(data["categories"]) + + # create datums + image_id_to_datum = { + image["id"]: _parse_image_to_datum(image) + for image in data["images"] + } + + # create groundtruths + for image in tqdm(data["annotations"], "Saving to JSON."): + # exract masks from annotations + mask_ids = _create_masks(masks_path / image["filename"]) + + # create annotations + annotations = create( + image, + category_id_to_labels_and_task, + mask_ids, + ) + gt = GroundTruth( + datum=image_id_to_datum[image["image_id"]], + annotations=annotations, + ) + f.write(json.dumps(gt.encode_value()).encode("utf-8")) + f.write("\n".encode("utf-8")) - """ - client = Client() + +def create_semantic_groundtruths_file( + dtype: AnnotationType, + path: Path, + filename: str, + destination: Path, + coco_url: str, + limit: int, +): + if dtype not in [ + AnnotationType.MULTIPOLYGON, + AnnotationType.RASTER, + ]: + raise ValueError(dtype) # download and unzip coco dataset + coco_path = path / destination data = download_coco_panoptic( - destination=Path(destination), + destination=coco_path, coco_url=coco_url, ) # path of mask locations - masks_path = destination / Path("annotations/panoptic_val2017/") + masks_path = coco_path / Path("annotations/panoptic_val2017/") # slice if limited if limit > 0: data["annotations"] = data["annotations"][:limit] - # if reset, delete the dataset if it exists - if delete_if_exists: - try: - client.delete_dataset(name, timeout=5) - except Exception: - pass - - # create groundtruths - gts = _create_groundtruths_from_coco_panoptic( - data=data, - masks_path=masks_path, - ) + # get filepath + filepath = path / Path(filename) - # extract metadata - metadata = data["info"].copy() - metadata["licenses"] = str(data["licenses"]) + with open(filepath, mode="wb") as f: - # create dataset - dataset = Dataset.create( - name, - metadata=metadata, - ) - for gt in tqdm(gts, desc="Uploading"): - dataset.add_groundtruth(gt) - dataset.finalize() + # extract labels from categories + category_id_to_labels_and_task = _parse_categories(data["categories"]) + + # create datums + image_id_to_datum = { + image["id"]: _parse_image_to_datum(image) + for image in data["images"] + } + + # create groundtruths + for image in tqdm(data["annotations"], "Saving to JSON."): + # exract masks from annotations + mask_ids = _create_masks(masks_path / image["filename"]) + + # create semantic segmentations + semantic_annotations = create_semantic_segmentations( + image, + category_id_to_labels_and_task, + mask_ids, + dtype=dtype, + ) + gt = GroundTruth( + datum=image_id_to_datum[image["image_id"]], + annotations=semantic_annotations, + ) + f.write(json.dumps(gt.encode_value()).encode("utf-8")) + f.write("\n".encode("utf-8")) + + +def get_instance_groundtruths( + dtype: AnnotationType, + chunk_size: int = 1, + limit: int = 0, + from_cache: bool = True, +) -> Generator[List[GroundTruth], None, None]: + """ + Retrieves COCO object detection groundtruths from a variety of sources. + + Parameters + ---------- + dtype : AnnotationType + The desired annotation type. + chunk_size : int, default=1 + The number of groundtruths returned per call. + limit : int, default=0 + The maximum number of groundtruths returned. Defaults to all. + from_cache : bool, default=True + Retrieve cached groundtruths rather than regenerate. + """ + + if dtype not in [ + AnnotationType.BOX, + AnnotationType.POLYGON, + AnnotationType.MULTIPOLYGON, + AnnotationType.RASTER, + ]: + raise ValueError(dtype) + + # paths + path = Path(os.path.dirname(os.path.realpath(__file__))) + coco_url = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip" + cache_url = "https://pub-fae71003f78140bdaedf32a7c8d331d2.r2.dev/" + + # get filename + filenames = { + AnnotationType.BOX: "gt_objdet_coco_bbox.jsonl", + AnnotationType.POLYGON: "gt_objdet_coco_polygon.jsonl", + AnnotationType.MULTIPOLYGON: "gt_objdet_coco_raster_multipolygon.jsonl", + AnnotationType.RASTER: "gt_objdet_coco_raster_bitmask.jsonl", + } + filename = filenames[dtype] + filepath = path / Path(filename) + + if from_cache: + download_data_if_not_exists( + filename=filename, + filepath=filepath, + url=f"{cache_url}{filename}", + ) + else: + create_instance_groundtruths_file( + dtype=dtype, + filename=filename, + path=path, + destination=Path("coco"), + coco_url=coco_url, + limit=limit, + ) + + with open(filepath, "r") as f: + count = 0 + chunks = [] + for line in f: + gt_dict = json.loads(line) + gt = GroundTruth.decode_value(gt_dict) + chunks.append(gt) + count += 1 + if count >= limit and limit > 0: + break + elif len(chunks) < chunk_size: + continue + + yield chunks + chunks = [] + if chunks: + yield chunks + + +def get_semantic_groundtruths( + dtype: AnnotationType, + chunk_size: int = 1, + limit: int = 0, + from_cache: bool = True, +) -> Generator[List[GroundTruth], None, None]: + """ + Retrieves COCO semantic segmenations groundtruths from a variety of sources. + + Parameters + ---------- + dtype : AnnotationType + The desired annotation type. + chunk_size : int, default=1 + The number of groundtruths returned per call. + limit : int, default=0 + The maximum number of groundtruths returned. Defaults to all. + from_cache : bool, default=True + Retrieve cached groundtruths rather than regenerate. + """ + + if dtype not in [ + AnnotationType.MULTIPOLYGON, + AnnotationType.RASTER, + ]: + raise ValueError(dtype) + + # paths + path = Path(os.path.dirname(os.path.realpath(__file__))) + coco_url = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip" + cache_url = "https://pub-fae71003f78140bdaedf32a7c8d331d2.r2.dev/" + + # get filename + + filenames = { + AnnotationType.MULTIPOLYGON: "gt_semseg_coco_raster_multipolygon.jsonl", + AnnotationType.RASTER: "gt_semseg_coco_raster_bitmask.jsonl", + } + filename = filenames[dtype] + filepath = path / Path(filename) + + if from_cache: + download_data_if_not_exists( + filename=filename, + filepath=filepath, + url=f"{cache_url}{filename}", + ) + else: + create_semantic_groundtruths_file( + dtype=dtype, + filename=filename, + path=path, + destination=Path("coco"), + coco_url=coco_url, + limit=limit, + ) - return dataset + with open(filepath, "r") as f: + count = 0 + chunks = [] + for line in f: + gt_dict = json.loads(line) + gt = GroundTruth.decode_value(gt_dict) + chunks.append(gt) + count += 1 + if count >= limit and limit > 0: + break + elif len(chunks) < chunk_size: + continue + + yield chunks + chunks = [] + if chunks: + yield chunks + + +if __name__ == "__main__": + + for chunk in get_instance_groundtruths( + dtype=AnnotationType.BOX, + chunk_size=2, + limit=8, + from_cache=True, + ): + print(chunk[0].datum.uid, chunk[1].datum.uid) diff --git a/examples/object-detection/integrations/yolo_integration.py b/examples/object-detection/integrations/yolo_integration.py index eed20c38e..34b419cf9 100644 --- a/examples/object-detection/integrations/yolo_integration.py +++ b/examples/object-detection/integrations/yolo_integration.py @@ -1,13 +1,213 @@ -import numpy -import PIL -from PIL.Image import Resampling +import json +import os +import re +import tempfile +import zipfile +from io import BytesIO +from pathlib import Path +from typing import Generator, List + +import numpy as np +import PIL.Image +import requests +import ultralytics +from shapely import geometry, ops +from skimage import measure +from tqdm import tqdm from valor import Annotation, Datum, Label, Prediction +from valor.enums import AnnotationType from valor.metatypes import ImageMetadata -from valor.schemas import Box, Raster +from valor.schemas import Box, MultiPolygon, Polygon, Raster + + +def download_data_if_not_exists( + filename: str, + filepath: Path, + url: str, +): + """Download the data from a public bucket if it doesn't exist locally.""" + + if not os.path.exists(filepath): + response = requests.get(url, stream=True) + if response.status_code == 200: + total_size = int(response.headers.get("content-length", 0)) + with open(filepath, "wb") as f: + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=filename, + ) as pbar: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + pbar.update(1024) + else: + raise RuntimeError(response) + else: + print(f"{filename} already exists locally.") + + +def download_coco_panoptic( + destination: Path = Path("./coco"), + coco_url: str = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip", +) -> dict: + """ + Download and return COCO panoptic dataset. + """ + + # append the location of the annotations within the destination folder + annotations_zipfile = destination / Path( + "annotations/panoptic_val2017.zip" + ) + + if not os.path.exists(str(destination)): + # Make a GET request to the URL + response = requests.get(coco_url, stream=True) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Get the total file size (if available) + total_size = int(response.headers.get("content-length", 0)) + + # Create a temporary file to save the downloaded content + with tempfile.TemporaryFile() as temp_file: + # Initialize tqdm with the total file size + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc="Downloading", + ) as pbar: + # Iterate over the response content and update progress + for chunk in response.iter_content(chunk_size=1024): + if chunk: + temp_file.write(chunk) + pbar.update(1024) + + # Once the file is downloaded, extract it + with zipfile.ZipFile(temp_file, "r") as zip_ref: + total_files = len(zip_ref.infolist()) + with tqdm( + total=total_files, unit="file", desc="Extracting" + ) as extraction_pbar: + for file_info in zip_ref.infolist(): + zip_ref.extract(file_info, str(destination)) + extraction_pbar.update(1) + # unzip the validation set + folder = str(annotations_zipfile.parent.absolute()) + filepath = str(annotations_zipfile.absolute()) + with zipfile.ZipFile(filepath, "r") as zip_ref: + zip_ref.extractall(folder) + else: + print(f"coco already exists at {destination}!") -def parse_detection_into_bounding_box( + with open(str(annotations_zipfile.with_suffix(".json"))) as f: + panoptic_val2017 = json.load(f) + + return panoptic_val2017 + + +def _convert_yolo_segmentation( + raw, + height: int, + width: int, + resample: PIL.Image.Resampling = PIL.Image.Resampling.BILINEAR, +): + """Resizes the raw binary mask provided by the YOLO inference to the original image size.""" + mask = np.asarray(raw.cpu()) + mask[mask == 1.0] = 255 + img = PIL.Image.fromarray(np.uint8(mask)) + img = img.resize((width, height), resample=resample) + mask = np.array(img, dtype=np.uint8) >= 128 + return mask + + +def _parse_image_to_datum(image: dict) -> Datum: + """ + Parse COCO image to Valor Datum + """ + image = image.copy() + uid = str(image.pop("id")) + height = image.pop("height") + width = image.pop("width") + image_metadata = ImageMetadata.create( + uid=uid, + height=height, + width=width, + metadata=image, + ) + return image_metadata.datum + + +def download_image(url: str) -> PIL.Image.Image: + """ + Download image using Datum. + """ + if not isinstance(url, str): + raise TypeError("datum.metadata['coco_url'] is not type 'str'.") + img_data = BytesIO(requests.get(url).content) + return PIL.Image.open(img_data) + + +def bitmask_to_multipolygon_raster(bitmask) -> Raster: + bitmask = np.array(bitmask, dtype=bool) + labeled_array, num_features = measure.label( + bitmask, background=0, return_num=True + ) + polygons = [] + for region_index in range(1, num_features + 1): + contours = measure.find_contours(labeled_array == region_index, 0.5) + for contour in contours: + if len(contour) >= 3: + polygon = geometry.Polygon(contour) + if polygon.is_valid: + polygons.append(polygon) + mp = geometry.MultiPolygon(polygons).simplify(tolerance=0.6) + values = [] + if isinstance(mp, geometry.MultiPolygon): + for polygon in mp.geoms: + boundary = list(polygon.exterior.coords) + holes = [list(interior.coords) for interior in polygon.interiors] + values.append([boundary, *holes]) + else: + boundary = list(mp.exterior.coords) + holes = [list(interior.coords) for interior in mp.interiors] + values = [[boundary, *holes]] + height, width = bitmask.shape + return Raster.from_geometry( + MultiPolygon(values), height=height, width=width + ) + + +def bitmask_to_polygon(bitmask) -> Polygon: + bitmask = np.array(bitmask, dtype=bool) + labeled_array, num_features = measure.label( + bitmask, background=0, return_num=True + ) + polygons = [] + for region_index in range(1, num_features + 1): + contours = measure.find_contours(labeled_array == region_index, 0.5) + for contour in contours: + if len(contour) >= 3: + polygon = geometry.Polygon(contour) + if polygon.is_valid: + polygons.append(polygon) + polygon = ops.unary_union( + geometry.MultiPolygon(polygons).simplify(tolerance=0.6) + ) + if not isinstance(polygon, geometry.Polygon): + return None + boundary = list(polygon.exterior.coords) + holes = [list(interior.coords) for interior in polygon.interiors] + return Polygon([boundary, *holes]) + + +def create_bounding_box_detection( result, datum: Datum, label_key: str = "class" ) -> Prediction: """Parses Ultralytic's result for an object detection task.""" @@ -16,7 +216,7 @@ def parse_detection_into_bounding_box( result = result[0] probabilities = [conf.item() for conf in result.boxes.conf] labels = [result.names[int(pred.item())] for pred in result.boxes.cls] - bboxes = [numpy.asarray(box.cpu()) for box in result.boxes.xyxy] + bboxes = [np.asarray(box.cpu()) for box in result.boxes.xyxy] # validate dimensions image_metadata = ImageMetadata(datum) @@ -55,26 +255,11 @@ def parse_detection_into_bounding_box( ) -def _convert_yolo_segmentation( - raw, - height: int, - width: int, - resample: Resampling = Resampling.BILINEAR, -): - """Resizes the raw binary mask provided by the YOLO inference to the original image size.""" - mask = numpy.asarray(raw.cpu()) - mask[mask == 1.0] = 255 - img = PIL.Image.fromarray(numpy.uint8(mask)) - img = img.resize((width, height), resample=resample) - mask = numpy.array(img, dtype=numpy.uint8) >= 128 - return mask - - -def parse_detection_into_raster( +def create_raster_detection( result, datum: Datum, label_key: str = "class", - resample: Resampling = Resampling.BILINEAR, + resample: PIL.Image.Resampling = PIL.Image.Resampling.BILINEAR, ) -> Prediction: """Parses Ultralytic's result for an image segmentation task.""" @@ -127,3 +312,236 @@ def parse_detection_into_raster( for mask, scored_label in list(zip(masks, labels)) ], ) + + +def create_multipolygon_detection( + result, + datum: Datum, + label_key: str = "class", + resample: PIL.Image.Resampling = PIL.Image.Resampling.BILINEAR, +): + prediction = create_raster_detection( + result=result, datum=datum, label_key=label_key, resample=resample + ) + annotations = [] + for annotation in prediction.annotations: + array = annotation.raster.array + multipolygon = bitmask_to_multipolygon_raster(array) + if multipolygon is not None: + annotation.raster = multipolygon + annotations.append(annotation) + prediction.annotations = annotations + return prediction + + +def create_polygon_detection( + result, + datum: Datum, + label_key: str = "class", + resample: PIL.Image.Resampling = PIL.Image.Resampling.BILINEAR, +): + prediction = create_raster_detection( + result=result, datum=datum, label_key=label_key, resample=resample + ) + annotations = [] + for annotation in prediction.annotations: + array = annotation.raster.array + polygon = bitmask_to_polygon(array) + if polygon is not None: + annotation.polygon = polygon + annotation.raster = None + annotations.append(annotation) + prediction.annotations = annotations + return prediction + + +def run_inference( + path: Path, + destination: Path, + coco_url: str, + limit: int, +): + """ + Creates Dataset and associated GroundTruths. + + Parameters + ---------- + name : str + Desired dataset name. + path : str + The working directory. + destination : str + Desired output path for dataset annotations. + coco_url : str + URL to the COCO dataset. + limit : int + Limits the number of datums. Set to 0 for no action. + """ + + coco_path = Path(path) / Path(destination) + + # download and unzip coco dataset + data = download_coco_panoptic( + destination=coco_path, + coco_url=coco_url, + ) + + # create datums + datums = [_parse_image_to_datum(image) for image in data["images"]] + if limit > 0 and limit < len(datums): + datums = datums[:limit] + + inference_engine = ultralytics.YOLO("yolov8n-seg.pt") + + filepath_bbox = Path(path) / Path("pd_objdet_yolo_bbox.jsonl") + filepath_polygon = Path(path) / Path("pd_objdet_yolo_polygon.jsonl") + filepath_multipolygon = Path(path) / Path( + "pd_objdet_yolo_multipolygon.jsonl" + ) + filepath_raster = Path(path) / Path("pd_objdet_yolo_raster.jsonl") + + with open(filepath_bbox, "w") as fbox: + with open(filepath_polygon, "w") as fpolygon: + with open(filepath_multipolygon, "w") as fmultipolygon: + with open(filepath_raster, "w") as fraster: + + for datum in tqdm(datums): + + image = download_image(datum.metadata["coco_url"]) + + results = inference_engine(image, verbose=False) + + # convert result into Valor Bounding Box prediction + prediction = create_bounding_box_detection( + results, # raw inference + datum=datum, # valor datum + label_key="name", # label_key override + ) + fbox.write(json.dumps(prediction.encode_value())) + fbox.write("\n") + + # convert result into Valor Bounding Polygon prediction + prediction = create_polygon_detection( + results, # raw inference + datum=datum, # valor datum + label_key="name", # label_key override + ) + fpolygon.write(json.dumps(prediction.encode_value())) + fpolygon.write("\n") + + # convert result into Valor MultiPolygon Raster prediction + prediction = create_multipolygon_detection( + results, # raw inference + datum=datum, # valor datum + label_key="name", # label_key override + ) + fmultipolygon.write( + json.dumps(prediction.encode_value()) + ) + fmultipolygon.write("\n") + + # convert result into Valor Bitmask Raster prediction + prediction = create_raster_detection( + results, # raw inference + datum=datum, # valor datum + label_key="name", # label_key override + ) + fraster.write(json.dumps(prediction.encode_value())) + fraster.write("\n") + + +def get_instance_predictions( + coco_uids: list[str], + dtype: AnnotationType, + chunk_size: int = 1, + limit: int = 0, + from_cache: bool = True, +) -> Generator[List[Prediction], None, None]: + """ + Retrieves YOLO object detection predictions. + + Parameters + ---------- + dtype : AnnotationType + The desired annotation type. + chunk_size : int, default=1 + The number of groundtruths returned per call. + limit : int, default=0 + The maximum number of groundtruths returned. Defaults to all. + from_cache : bool, default=True + Retrieve cached groundtruths rather than regenerate. + """ + + # paths + path = Path(os.path.dirname(os.path.realpath(__file__))) + coco_url = "http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip" + cache_url = "https://pub-fae71003f78140bdaedf32a7c8d331d2.r2.dev/" + + # get filename + filenames = { + AnnotationType.BOX: "pd_objdet_yolo_bbox.jsonl", + AnnotationType.POLYGON: "pd_objdet_yolo_polygon.jsonl", + AnnotationType.MULTIPOLYGON: "pd_objdet_yolo_multipolygon.jsonl", + AnnotationType.RASTER: "pd_objdet_yolo_raster.jsonl", + } + filename = filenames[dtype] + filepath = path / Path(filename) + + if from_cache: + download_data_if_not_exists( + filename=filename, + filepath=filepath, + url=f"{cache_url}{filename}", + ) + else: + run_inference( + path=path, + destination=Path("coco"), + coco_url=coco_url, + limit=limit, + ) + + with open(filepath, "r") as f: + pattern = re.compile(r'"uid":\s*"(\d+)"') + count = 0 + chunks = [] + for line in f: + match = pattern.search(line) + if not match: + continue + elif match.group(1) not in coco_uids: + continue + pd_dict = json.loads(line) + pd = Prediction.decode_value(pd_dict) + chunks.append(pd) + count += 1 + if count >= limit and limit > 0: + break + elif len(chunks) < chunk_size: + continue + + yield chunks + chunks = [] + if chunks: + yield chunks + + +if __name__ == "__main__": + + for chunk in get_instance_predictions( + coco_uids=[ + "139", + "285", + "632", + "724", + "776", + "785", + "802", + "872", + ], + dtype=AnnotationType.BOX, + chunk_size=2, + limit=8, + from_cache=True, + ): + print(chunk[0].datum.uid, chunk[1].datum.uid) diff --git a/integration_tests/benchmarks/.gitignore b/integration_tests/benchmarks/.gitignore index 94a2dd146..945e26b14 100644 --- a/integration_tests/benchmarks/.gitignore +++ b/integration_tests/benchmarks/.gitignore @@ -1 +1,2 @@ -*.json \ No newline at end of file +*.json +*.jsonl \ No newline at end of file diff --git a/integration_tests/benchmarks/classification/benchmark_script.py b/integration_tests/benchmarks/classification/benchmark_script.py index 1d644b84a..c1d58ec7b 100644 --- a/integration_tests/benchmarks/classification/benchmark_script.py +++ b/integration_tests/benchmarks/classification/benchmark_script.py @@ -1,7 +1,9 @@ import json import os import time +from dataclasses import dataclass from datetime import datetime +from pathlib import Path import requests @@ -22,7 +24,7 @@ client = Client() -def download_data_if_not_exists(file_path: str, file_url: str): +def download_data_if_not_exists(file_path: Path, file_url: str): """Download the data from a public bucket if it doesn't exist in the repo.""" if os.path.exists(file_path): return @@ -32,10 +34,9 @@ def download_data_if_not_exists(file_path: str, file_url: str): json.dump(response, file, indent=4) -def write_results_to_file(write_path: str, result_dict: dict): +def write_results_to_file(write_path: Path, results: list[dict]): """Write results to results.json""" current_datetime = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - if os.path.isfile(write_path): with open(write_path, "r") as file: file.seek(0) @@ -43,25 +44,27 @@ def write_results_to_file(write_path: str, result_dict: dict): else: data = {} - data[current_datetime] = result_dict + data[current_datetime] = results with open(write_path, "w+") as file: json.dump(data, file, indent=4) -def ingest_groundtruths_and_predictions( - dset: Dataset, model: Model, raw: dict, pair_limit: int +def ingest_groundtruths( + dset: Dataset, + raw: dict, + pair_limit: int, + timeout: int | None, ): - """Ingest the data into Valor.""" + """Ingest groundtruths into Valor.""" groundtruths = [] - predictions = [] slice_ = ( raw["groundtruth_prediction_pairs"][:pair_limit] if pair_limit != -1 else raw["groundtruth_prediction_pairs"] ) - for groundtruth, prediction in slice_: + for groundtruth, _ in slice_: groundtruths.append( GroundTruth( datum=Datum( @@ -84,6 +87,26 @@ def ingest_groundtruths_and_predictions( ) ) + dset.add_groundtruths(groundtruths, timeout=timeout) + + +def ingest_predictions( + dset: Dataset, + model: Model, + raw: dict, + pair_limit: int, + timeout: int | None, +): + """Ingest the data into Valor.""" + + predictions = [] + slice_ = ( + raw["groundtruth_prediction_pairs"][:pair_limit] + if pair_limit != -1 + else raw["groundtruth_prediction_pairs"] + ) + for _, prediction in slice_: + predictions.append( Prediction( datum=Datum( @@ -106,65 +129,122 @@ def ingest_groundtruths_and_predictions( ) ) - dset.add_groundtruths(groundtruths, timeout=150) - model.add_predictions(dset, predictions, timeout=150) + model.add_predictions(dset, predictions, timeout=timeout) - dset.finalize() - model.finalize_inferences(dataset=dset) - -def run_base_evaluation(dset: Dataset, model: Model): +def run_base_evaluation(dset: Dataset, model: Model, timeout: int | None): """Run a base evaluation (with no PR curves).""" - evaluation = model.evaluate_classification(dset) - evaluation.wait_for_completion() + try: + evaluation = model.evaluate_classification(dset) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"Base evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore + ) return evaluation -def run_pr_curve_evaluation(dset: Dataset, model: Model): +def run_pr_curve_evaluation(dset: Dataset, model: Model, timeout: int | None): """Run a base evaluation with PrecisionRecallCurve included.""" - evaluation = model.evaluate_classification( - dset, - metrics_to_return=[ - MetricType.Accuracy, - MetricType.Precision, - MetricType.Recall, - MetricType.F1, - MetricType.ROCAUC, - MetricType.PrecisionRecallCurve, - ], - ) - evaluation.wait_for_completion() + try: + evaluation = model.evaluate_classification( + dset, + metrics_to_return=[ + MetricType.Accuracy, + MetricType.Precision, + MetricType.Recall, + MetricType.F1, + MetricType.ROCAUC, + MetricType.PrecisionRecallCurve, + ], + ) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"PR evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore + ) return evaluation -def run_detailed_pr_curve_evaluation(dset: Dataset, model: Model): +def run_detailed_pr_curve_evaluation( + dset: Dataset, model: Model, timeout: int | None +): """Run a base evaluation with PrecisionRecallCurve and DetailedPrecisionRecallCurve included.""" - evaluation = model.evaluate_classification( - dset, - metrics_to_return=[ - MetricType.Accuracy, - MetricType.Precision, - MetricType.Recall, - MetricType.F1, - MetricType.ROCAUC, - MetricType.PrecisionRecallCurve, - MetricType.DetailedPrecisionRecallCurve, - ], - ) - evaluation.wait_for_completion() + try: + evaluation = model.evaluate_classification( + dset, + metrics_to_return=[ + MetricType.Accuracy, + MetricType.Precision, + MetricType.Recall, + MetricType.F1, + MetricType.ROCAUC, + MetricType.PrecisionRecallCurve, + MetricType.DetailedPrecisionRecallCurve, + ], + ) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"Detailed evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore + ) return evaluation +@dataclass +class DataBenchmark: + ingestion: float + finalization: float + deletion: float + + def result(self) -> dict[str, float | str]: + return { + "ingestion": round(self.ingestion, 2), + "finalization": round(self.finalization, 2), + "deletion": round(self.deletion, 2), + } + + +@dataclass +class EvaluationBenchmark: + limit: int + gt_stats: DataBenchmark + pd_stats: DataBenchmark + n_datums: int + n_annotations: int + n_labels: int + eval_base: float + eval_base_pr: float + eval_base_pr_detail: float + + def result(self) -> dict[str, float | str | dict[str, str | float]]: + return { + "limit": self.limit, + "groundtruths": self.gt_stats.result(), + "predictions": self.pd_stats.result(), + "evaluation": { + "number_of_datums": self.n_datums, + "number_of_annotations": self.n_annotations, + "number_of_labels": self.n_labels, + "base": round(self.eval_base, 2), + "base+pr": round(self.eval_base_pr, 2), + "base+pr+detailed": round(self.eval_base_pr_detail, 2), + }, + } + + def run_benchmarking_analysis( - limits_to_test: list[int] = [5000, 5000], + limits: list[int], results_file: str = "results.json", data_file: str = "data.json", + ingestion_timeout: int | None = 150, + evaluation_timeout: int | None = 40, ): """Time various function calls and export the results.""" - current_directory = os.path.dirname(os.path.realpath(__file__)) - write_path = f"{current_directory}/{results_file}" - data_path = f"{current_directory}/{data_file}" + current_directory = Path(os.path.dirname(os.path.realpath(__file__))) + write_path = current_directory / Path(results_file) + data_path = current_directory / Path(data_file) download_data_if_not_exists( file_path=data_path, @@ -175,58 +255,88 @@ def run_benchmarking_analysis( file.seek(0) raw_data = json.load(file) - for limit in limits_to_test: + results = list() + for limit in limits: dset = Dataset.create(name=f"bird-identification-{time.time()}") model = Model.create(name=f"some_model-{time.time()}") + # ingest groundtruths start_time = time.time() + ingest_groundtruths( + dset=dset, + raw=raw_data, + pair_limit=limit, + timeout=ingestion_timeout, + ) + gt_ingest_time = time.time() - start_time - ingest_groundtruths_and_predictions( - dset=dset, model=model, raw=raw_data, pair_limit=limit + # finalize groundtruths + start_time = time.time() + dset.finalize() + gt_finalization_time = time.time() - start_time + + # ingest predictions + start_time = time.time() + ingest_predictions( + dset=dset, + model=model, + raw=raw_data, + pair_limit=limit, + timeout=ingestion_timeout, ) - ingest_time = time.time() - start_time + pd_ingest_time = time.time() - start_time - try: - eval_ = run_base_evaluation(dset=dset, model=model) - except TimeoutError: - raise TimeoutError( - f"Evaluation timed out when processing {limit} datums." - ) + # finalize predictions + start_time = time.time() + model.finalize_inferences(dset) + pd_finalization_time = time.time() - start_time - try: - eval_pr = run_pr_curve_evaluation(dset=dset, model=model) - except TimeoutError: - raise TimeoutError( - f"PR Evaluation timed out when processing {limit} datums." - ) + # run evaluations + eval_base = run_base_evaluation( + dset=dset, model=model, timeout=evaluation_timeout + ) + eval_pr = run_pr_curve_evaluation( + dset=dset, model=model, timeout=evaluation_timeout + ) + eval_detail = run_detailed_pr_curve_evaluation( + dset=dset, model=model, timeout=evaluation_timeout + ) - try: - eval_pr_detail = run_detailed_pr_curve_evaluation( - dset=dset, model=model - ) - except TimeoutError: - raise TimeoutError( - f"Detailed PR Evaluation timed out when processing {limit} datums." - ) + # delete model + start = time.time() + client.delete_model(model.name, timeout=30) + pd_deletion_time = time.time() - start + # delete dataset start = time.time() client.delete_dataset(dset.name, timeout=30) - client.delete_model(model.name, timeout=30) - deletion_time = time.time() - start - - results = { - "number_of_datums": limit, - "number_of_unique_labels": eval_.meta["labels"], - "number_of_annotations": eval_.meta["annotations"], - "ingest_runtime": f"{(ingest_time):.1f} seconds", - "eval_runtime": f"{(eval_.meta['duration']):.1f} seconds", - "eval_pr_runtime": f"{(eval_pr.meta['duration']):.1f} seconds", - "eval_detailed_pr_runtime": f"{(eval_pr_detail.meta['duration']):.1f} seconds", - "del_runtime": f"{(deletion_time):.1f} seconds", - } - write_results_to_file(write_path=write_path, result_dict=results) + gt_deletion_time = time.time() - start + + results.append( + EvaluationBenchmark( + limit=limit, + gt_stats=DataBenchmark( + ingestion=gt_ingest_time, + finalization=gt_finalization_time, + deletion=gt_deletion_time, + ), + pd_stats=DataBenchmark( + ingestion=pd_ingest_time, + finalization=pd_finalization_time, + deletion=pd_deletion_time, + ), + n_datums=eval_base.meta["datums"], + n_annotations=eval_base.meta["annotations"], + n_labels=eval_base.meta["labels"], + eval_base=eval_base.meta["duration"], + eval_base_pr=eval_pr.meta["duration"], + eval_base_pr_detail=eval_detail.meta["duration"], + ).result() + ) + + write_results_to_file(write_path=write_path, results=results) if __name__ == "__main__": - run_benchmarking_analysis() + run_benchmarking_analysis(limits=[5000, 5000]) diff --git a/integration_tests/benchmarks/object-detection/benchmark_script.py b/integration_tests/benchmarks/object-detection/benchmark_script.py index 9258148ed..a602ea7bd 100644 --- a/integration_tests/benchmarks/object-detection/benchmark_script.py +++ b/integration_tests/benchmarks/object-detection/benchmark_script.py @@ -1,48 +1,60 @@ import json import os +import re +from dataclasses import dataclass from datetime import datetime +from pathlib import Path from time import time import requests +from tqdm import tqdm -from valor import ( - Annotation, - Client, - Dataset, - Datum, - GroundTruth, - Label, - Model, - Prediction, - connect, -) -from valor.schemas import MultiPolygon, Polygon, Raster +from valor import Client, Dataset, GroundTruth, Model, Prediction, connect +from valor.enums import AnnotationType +from valor.exceptions import DatasetAlreadyExistsError, ModelAlreadyExistsError connect("http://0.0.0.0:8000") client = Client() -def download_data_if_not_exists(file_path: str, file_url: str): - """Download the data from a public bucket if it doesn't exist in the repo.""" - if os.path.exists(file_path): - return +def time_it(fn, *args, **kwargs): + start = time() + fn(*args, **kwargs) + return time() - start - response = json.loads(requests.get(file_url).text) - with open(file_path, "w+") as file: - json.dump(response, file, indent=4) - -def _convert_wkt_to_coordinates(wkt: str) -> list[list[tuple]]: - """Convert a WKT string into a nested list of coordinates.""" - return [ - [tuple(float(y) for y in x) for x in json.loads(wkt)["coordinates"][0]] - ] +def download_data_if_not_exists( + file_name: str, + file_path: Path, + url: str, +): + """Download the data from a public bucket if it doesn't exist locally.""" + + if not os.path.exists(file_path): + response = requests.get(url, stream=True) + if response.status_code == 200: + total_size = int(response.headers.get("content-length", 0)) + with open(file_path, "wb") as f: + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=file_name, + ) as pbar: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + pbar.update(1024) + else: + raise RuntimeError(response) + else: + print(f"{file_name} already exists locally.") -def write_results_to_file(write_path: str, result_dict: dict): +def write_results_to_file(write_path: Path, results: list[dict]): """Write results to results.json""" current_datetime = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - if os.path.isfile(write_path): with open(write_path, "r") as file: file.seek(0) @@ -50,243 +62,364 @@ def write_results_to_file(write_path: str, result_dict: dict): else: data = {} - data[current_datetime] = result_dict + data[current_datetime] = results with open(write_path, "w+") as file: json.dump(data, file, indent=4) -def ingest_groundtruths_and_predictions( - dset: Dataset, model: Model, raw: list, pair_limit: int +def ingest_groundtruths( + dataset: Dataset, + path: Path, + limit: int, + chunk_size: int, + timeout: int | None, ): - """Ingest the data into Valor.""" - groundtruths = [] - predictions = [] - - for datum_id, data in raw[:pair_limit]: - datum = Datum( - uid=str(datum_id), - metadata=data["datum_metadata"], - ) - groundtruths.append( - GroundTruth( - datum=datum, - annotations=list( - [ - Annotation( - is_instance=ann["is_instance"], - labels=list( - [ - Label( - key=label["key"], - value=label["value"], - ) - for label in ann["labels"] - ] - ), - bounding_box=( - _convert_wkt_to_coordinates(ann["box"]) - if ann["box"] - else None - ), - raster=( - Raster.from_geometry( - geometry=MultiPolygon( - [ - _convert_wkt_to_coordinates( - ann["raster"] - ) - ] - ), - height=data["datum_metadata"]["height"], - width=data["datum_metadata"]["width"], - ) - if ann["raster"] - else None - ), - polygon=( - ( - Polygon( - _convert_wkt_to_coordinates( - ann["polygon"] - ) - ) - ) - if ann["polygon"] - else None - ), - ) - for ann in data["groundtruth_annotations"] - ] - ), - ) - ) - - predictions.append( - Prediction( - datum=datum, - annotations=list( - [ - Annotation( - is_instance=ann["is_instance"], - labels=list( - [ - Label( - key=label["key"], - value=label["value"], - score=label["score"], - ) - for label in ann["labels"] - ] - ), - bounding_box=( - _convert_wkt_to_coordinates(ann["box"]) - if ann["box"] - else None - ), - raster=( - Raster.from_geometry( - geometry=MultiPolygon( - [ - _convert_wkt_to_coordinates( - ann["raster"] - ) - ] - ), - height=data["datum_metadata"]["height"], - width=data["datum_metadata"]["width"], - ) - if ann["raster"] - else None - ), - polygon=( - ( - Polygon( - _convert_wkt_to_coordinates( - ann["polygon"] - ) - ) - ) - if ann["polygon"] - else None - ), - ) - for ann in data["prediction_annotations"] - ] - ), - ) + with open(path, "r") as f: + count = 0 + chunks = [] + for line in f: + gt_dict = json.loads(line) + gt = GroundTruth.decode_value(gt_dict) + chunks.append(gt) + count += 1 + if count >= limit and limit > 0: + break + elif len(chunks) < chunk_size: + continue + + dataset.add_groundtruths(chunks, timeout=timeout) + chunks = [] + if chunks: + dataset.add_groundtruths(chunks, timeout=timeout) + + +def ingest_predictions( + dataset: Dataset, + model: Model, + datum_uids: list[str], + path: Path, + limit: int, + chunk_size: int, + timeout: int | None, +): + pattern = re.compile(r'"uid":\s*"(\d+)"') + with open(path, "r") as f: + count = 0 + chunks = [] + for line in f: + match = pattern.search(line) + if not match: + continue + elif match.group(1) not in datum_uids: + continue + pd_dict = json.loads(line) + pd = Prediction.decode_value(pd_dict) + chunks.append(pd) + count += 1 + if count >= limit and limit > 0: + break + elif len(chunks) < chunk_size: + continue + + model.add_predictions(dataset, chunks, timeout=timeout) + chunks = [] + if chunks: + model.add_predictions(dataset, chunks, timeout=timeout) + + +def run_base_evaluation(dset: Dataset, model: Model, timeout: int | None): + """Run a base evaluation (with no PR curves).""" + try: + evaluation = model.evaluate_detection(dset) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"Base evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore ) + return evaluation - for gt in groundtruths: - dset.add_groundtruth(gt) - for pred in predictions: - model.add_prediction(dset, pred) +def run_pr_curve_evaluation(dset: Dataset, model: Model, timeout: int | None): + """Run a base evaluation with PrecisionRecallCurve included.""" + try: + evaluation = model.evaluate_detection( + dset, + metrics_to_return=[ + "AP", + "AR", + "mAP", + "APAveragedOverIOUs", + "mAR", + "mAPAveragedOverIOUs", + "PrecisionRecallCurve", + ], + ) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"PR evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore + ) + return evaluation - dset.finalize() - model.finalize_inferences(dataset=dset) +def run_detailed_pr_curve_evaluation( + dset: Dataset, model: Model, timeout: int | None +): + """Run a base evaluation with PrecisionRecallCurve and DetailedPrecisionRecallCurve included.""" -def run_base_evaluation(dset: Dataset, model: Model): - """Run a base evaluation (with no PR curves).""" - evaluation = model.evaluate_detection(dset) - evaluation.wait_for_completion(timeout=60) + try: + evaluation = model.evaluate_detection( + dset, + metrics_to_return=[ + "AP", + "AR", + "mAP", + "APAveragedOverIOUs", + "mAR", + "mAPAveragedOverIOUs", + "PrecisionRecallCurve", + "DetailedPrecisionRecallCurve", + ], + ) + evaluation.wait_for_completion(timeout=timeout) + except TimeoutError: + raise TimeoutError( + f"Detailed evaluation timed out when processing {evaluation.meta['datums']} datums." # type: ignore + ) return evaluation -def run_pr_curve_evaluation(dset: Dataset, model: Model): - """Run a base evaluation with PrecisionRecallCurve included.""" - evaluation = model.evaluate_detection( - dset, - metrics_to_return=[ - "AP", - "AR", - "mAP", - "APAveragedOverIOUs", - "mAR", - "mAPAveragedOverIOUs", - "PrecisionRecallCurve", - ], - ) - evaluation.wait_for_completion() - return evaluation +@dataclass +class DataBenchmark: + dtype: str + ingestion: float + finalization: float + deletion: float + def result(self) -> dict[str, float | str]: + return { + "dtype": self.dtype, + "ingestion": round(self.ingestion, 2), + "finalization": round(self.finalization, 2), + "deletion": round(self.deletion, 2), + } -def run_detailed_pr_curve_evaluation(dset: Dataset, model: Model): - """Run a base evaluation with PrecisionRecallCurve and DetailedPrecisionRecallCurve included.""" - evaluation = model.evaluate_detection( - dset, - metrics_to_return=[ - "AP", - "AR", - "mAP", - "APAveragedOverIOUs", - "mAR", - "mAPAveragedOverIOUs", - "PrecisionRecallCurve", - "DetailedPrecisionRecallCurve", - ], - ) - evaluation.wait_for_completion() - return evaluation +@dataclass +class EvaluationBenchmark: + limit: int + gt_stats: DataBenchmark + pd_stats: DataBenchmark + n_datums: int + n_annotations: int + n_labels: int + eval_base: float + eval_base_pr: float + eval_base_pr_detail: float + + def result(self) -> dict[str, float | str | dict[str, str | float]]: + return { + "limit": self.limit, + "groundtruths": self.gt_stats.result(), + "predictions": self.pd_stats.result(), + "evaluation": { + "number_of_datums": self.n_datums, + "number_of_annotations": self.n_annotations, + "number_of_labels": self.n_labels, + "base": round(self.eval_base, 2), + "base+pr": round(self.eval_base_pr, 2), + "base+pr+detailed": round(self.eval_base_pr_detail, 2), + }, + } def run_benchmarking_analysis( - limits_to_test: list[int] = [6, 6], + limits_to_test: list[int], + combinations: list[tuple[AnnotationType, AnnotationType]] | None = None, results_file: str = "results.json", - data_file: str = "data.json", + ingestion_chunk_timeout: int = 30, + evaluation_timeout: int = 30, + compute_pr: bool = True, + compute_detailed: bool = True, ): """Time various function calls and export the results.""" - current_directory = os.path.dirname(os.path.realpath(__file__)) - write_path = f"{current_directory}/{results_file}" - data_path = f"{current_directory}/{data_file}" - - download_data_if_not_exists( - file_path=data_path, - file_url="https://pub-fae71003f78140bdaedf32a7c8d331d2.r2.dev/detection_data.json", - ) - - with open(data_path) as file: - file.seek(0) - raw_data = json.load(file) + current_directory = Path(__file__).parent + write_path = current_directory / Path(results_file) + + gt_box_filename = "gt_objdet_coco_bbox.jsonl" + gt_polygon_filename = "gt_objdet_coco_polygon.jsonl" + gt_multipolygon_filename = "gt_objdet_coco_raster_multipolygon.jsonl" + gt_raster_filename = "gt_objdet_coco_raster_bitmask.jsonl" + pd_box_filename = "pd_objdet_yolo_bbox.jsonl" + pd_polygon_filename = "pd_objdet_yolo_polygon.jsonl" + pd_multipolygon_filename = "pd_objdet_yolo_multipolygon.jsonl" + pd_raster_filename = "pd_objdet_yolo_raster.jsonl" + + groundtruths = { + AnnotationType.BOX: gt_box_filename, + AnnotationType.POLYGON: gt_polygon_filename, + AnnotationType.MULTIPOLYGON: gt_multipolygon_filename, + AnnotationType.RASTER: gt_raster_filename, + } + predictions = { + AnnotationType.BOX: pd_box_filename, + AnnotationType.POLYGON: pd_polygon_filename, + AnnotationType.MULTIPOLYGON: pd_multipolygon_filename, + AnnotationType.RASTER: pd_raster_filename, + } + + # default is to perform all combinations + if combinations is None: + combinations = [ + (gt_type, pd_type) + for gt_type in groundtruths + for pd_type in predictions + ] + + # cache data locally + filenames = [*list(groundtruths.values()), *list(predictions.values())] + for filename in filenames: + file_path = current_directory / Path(filename) + url = f"https://pub-fae71003f78140bdaedf32a7c8d331d2.r2.dev/{filename}" + download_data_if_not_exists( + file_name=filename, file_path=file_path, url=url + ) + # iterate through datum limits + results = list() for limit in limits_to_test: - dset = Dataset.create(name="coco-dataset") - model = Model.create(name="coco-model") - - # convert dict into list of tuples so we can slice it - raw_data_tuple = [(key, value) for key, value in raw_data.items()] + for gt_type, pd_type in combinations: + + gt_filename = groundtruths[gt_type] + pd_filename = predictions[pd_type] + + try: + dataset = Dataset.create(name="coco") + model = Model.create(name="yolo") + except ( + DatasetAlreadyExistsError, + ModelAlreadyExistsError, + ) as e: + client.delete_dataset("coco") + client.delete_model("yolo") + raise e + + # gt ingestion + gt_ingest_time = time_it( + ingest_groundtruths, + dataset=dataset, + path=current_directory / Path(gt_filename), + limit=limit, + chunk_size=1000, + timeout=ingestion_chunk_timeout, + ) - start_time = time() + # gt finalization + gt_finalization_time = time_it(dataset.finalize) + + # pd ingestion + datum_uids = [datum.uid for datum in dataset.get_datums()] + pd_ingest_time = time_it( + ingest_predictions, + dataset=dataset, + model=model, + datum_uids=datum_uids, + path=current_directory / Path(pd_filename), + limit=limit, + chunk_size=1000, + timeout=ingestion_chunk_timeout, + ) - ingest_groundtruths_and_predictions( - dset=dset, model=model, raw=raw_data_tuple, pair_limit=limit - ) - ingest_time = time() - start_time + # model finalization + pd_finalization_time = time_it(model.finalize_inferences, dataset) - try: - eval_ = run_base_evaluation(dset=dset, model=model) - except TimeoutError: - raise TimeoutError( - f"Evaluation timed out when processing {limit} datums." + # run evaluations + eval_pr = None + eval_detail = None + eval_base = run_base_evaluation( + dset=dataset, model=model, timeout=evaluation_timeout + ) + if compute_pr: + eval_pr = run_pr_curve_evaluation( + dset=dataset, model=model, timeout=evaluation_timeout + ) + if compute_detailed: + eval_detail = run_detailed_pr_curve_evaluation( + dset=dataset, model=model, timeout=evaluation_timeout + ) + + # delete model + start = time() + client.delete_model(model.name, timeout=30) + pd_deletion_time = time() - start + + # delete dataset + start = time() + client.delete_dataset(dataset.name, timeout=30) + gt_deletion_time = time() - start + + results.append( + EvaluationBenchmark( + limit=limit, + gt_stats=DataBenchmark( + dtype=gt_type, + ingestion=gt_ingest_time, + finalization=gt_finalization_time, + deletion=gt_deletion_time, + ), + pd_stats=DataBenchmark( + dtype=pd_type, + ingestion=pd_ingest_time, + finalization=pd_finalization_time, + deletion=pd_deletion_time, + ), + n_datums=eval_base.meta["datums"], + n_annotations=eval_base.meta["annotations"], + n_labels=eval_base.meta["labels"], + eval_base=eval_base.meta["duration"], + eval_base_pr=eval_pr.meta["duration"] if eval_pr else -1, + eval_base_pr_detail=( + eval_detail.meta["duration"] if eval_detail else -1 + ), + ).result() ) - start = time() - client.delete_dataset(dset.name, timeout=30) - client.delete_model(model.name, timeout=30) - deletion_time = time() - start - - results = { - "number_of_datums": limit, - "number_of_unique_labels": eval_.meta["labels"], - "number_of_annotations": eval_.meta["annotations"], - "ingest_runtime": f"{(ingest_time):.1f} seconds", - "eval_runtime": f"{(eval_.meta['duration']):.1f} seconds", - "del_runtime": f"{(deletion_time):.1f} seconds", - } - write_results_to_file(write_path=write_path, result_dict=results) + write_results_to_file(write_path=write_path, results=results) if __name__ == "__main__": - run_benchmarking_analysis() + + # run bounding box benchmark + run_benchmarking_analysis( + combinations=[ + (AnnotationType.BOX, AnnotationType.BOX), + ], + limits_to_test=[5000, 5000], + ) + + # run polygon benchmark + run_benchmarking_analysis( + combinations=[ + (AnnotationType.POLYGON, AnnotationType.POLYGON), + ], + limits_to_test=[5000, 5000], + ) + + # run multipolygon benchmark + run_benchmarking_analysis( + combinations=[ + (AnnotationType.MULTIPOLYGON, AnnotationType.MULTIPOLYGON), + ], + limits_to_test=[6, 6], + compute_detailed=False, + ) + + # run raster benchmark + run_benchmarking_analysis( + combinations=[ + (AnnotationType.RASTER, AnnotationType.RASTER), + ], + limits_to_test=[6, 6], + compute_detailed=False, + )