diff --git a/.gitignore b/.gitignore index 875766b1..3c74cb3d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +*.pyc # PyInstaller # Usually these files are written by a python script from a template diff --git a/src/av2/evaluation/detection/eval.py b/src/av2/evaluation/detection/eval.py index 2a8357a1..5bd4221c 100644 --- a/src/av2/evaluation/detection/eval.py +++ b/src/av2/evaluation/detection/eval.py @@ -58,6 +58,7 @@ import numpy as np import pandas as pd +from tqdm import tqdm from av2.evaluation.detection.constants import ( HIERARCHY, LCA, @@ -442,14 +443,14 @@ def evaluate_hierarchy( gts_categories_list.append(sweep_gts_categories) num_dts = len(sweep_dts) - num_gts = len(sweep_dts) + num_gts = len(sweep_gts) dts_uuids_list.extend(num_dts * [uuid]) gts_uuids_list.extend(num_gts * [uuid]) - dts_npy = np.concatenate(dts).astype(np.float64) - gts_npy = np.concatenate(gts).astype(np.float64) - dts_categories_npy = np.concatenate(dts_categories).astype(np.object_) - gts_categories_npy = np.concatenate(gts_categories).astype(np.object_) + dts_npy = np.concatenate(dts_list).astype(np.float64) + gts_npy = np.concatenate(gts_list).astype(np.float64) + dts_categories_npy = np.concatenate(dts_categories_list).astype(np.object_) + gts_categories_npy = np.concatenate(gts_categories_list).astype(np.object_) dts_uuids_npy = np.array(dts_uuids_list) gts_uuids_npy = np.array(gts_uuids_list) @@ -487,8 +488,9 @@ def evaluate_hierarchy( ) logger.info("Starting evaluation ...") - with mp.get_context("spawn").Pool(processes=n_jobs) as p: - accumulate_outputs: Any = p.starmap(accumulate_hierarchy, accumulate_hierarchy_args_list) + accumulate_outputs = [] + for accumulate_args in tqdm(accumulate_hierarchy_args_list): + accumulate_outputs.append(accumulate_hierarchy(*accumulate_args)) super_categories = list(HIERARCHY.keys()) metrics = np.zeros((len(cfg.categories), len(HIERARCHY.keys()))) diff --git a/src/av2/evaluation/detection/utils.py b/src/av2/evaluation/detection/utils.py index 5922e603..f6e6a136 100644 --- a/src/av2/evaluation/detection/utils.py +++ b/src/av2/evaluation/detection/utils.py @@ -279,7 +279,7 @@ def accumulate_hierarchy( fp: Dict[int, Any] = {} gt_name: Dict[int, List[Any]] = {} pred_name: Dict[int, List[Any]] = {} - taken: Dict[int, Set[Tuple[Any, Any]]] = {} + taken: Dict[int, Set[Tuple[Any, Any, Any]]] = {} for i in range(len(cfg.affinity_thresholds_m)): tp[i] = [] fp[i] = [] @@ -292,7 +292,11 @@ def accumulate_hierarchy( min_dist = len(cfg.affinity_thresholds_m) * [np.inf] match_gt_idx = len(cfg.affinity_thresholds_m) * [None] - keep_sweep = gts_uuids == np.array([gts.shape[0] * [pred_uuid]]).squeeze() + if len(gts_uuids) > 0: + keep_sweep = np.all(gts_uuids == np.array([gts.shape[0] * [pred_uuid]]).squeeze(), axis=1) + else: + keep_sweep = [] + gt_ind_sweep = np.arange(gts.shape[0])[keep_sweep] gts_sweep = gts[keep_sweep] gts_cats_sweep = gts_cats[keep_sweep] @@ -303,7 +307,7 @@ def accumulate_hierarchy( # Find closest match among ground truth boxes for i in range(len(cfg.affinity_thresholds_m)): - if gt_cat == cat and not (pred_uuid, gt_idx) in taken[i]: + if gt_cat == cat and not (pred_uuid[0], pred_uuid[1], gt_idx) in taken[i]: this_distance = dist_mat[pred_idx][gt_idx] if this_distance < min_dist[i]: min_dist[i] = this_distance @@ -316,7 +320,7 @@ def accumulate_hierarchy( # Find closest match among ground truth boxes for i in range(len(cfg.affinity_thresholds_m)): - if not is_match[i] and not (pred_uuid, gt_idx) in taken[i]: + if not is_match[i] and not (pred_uuid[0], pred_uuid[1], gt_idx) in taken[i]: this_distance = dist_mat[pred_idx][gt_idx] if this_distance < min_dist[i]: min_dist[i] = this_distance @@ -330,7 +334,7 @@ def accumulate_hierarchy( for i in range(len(cfg.affinity_thresholds_m)): if is_match[i]: - taken[i].add((pred_uuid, gt_idx)) + taken[i].add((pred_uuid[0], pred_uuid[1], gt_idx)) tp[i].append(1) fp[i].append(0) diff --git a/src/av2/evaluation/forecasting/SUBMISSION_FORMAT.md b/src/av2/evaluation/forecasting/SUBMISSION_FORMAT.md deleted file mode 100644 index e4c56d27..00000000 --- a/src/av2/evaluation/forecasting/SUBMISSION_FORMAT.md +++ /dev/null @@ -1,94 +0,0 @@ -# Forecasting Submission Format - -The evaluation expects a dictionary of dictionaries of lists of dictionaries - -```python -{ - : { - : [ - { - "prediction": - "score": - "detection_score": , - "instance_id": - "current_translation": , - "label":