From b2bb0609127bacf003d28850c284a9a7574cd039 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 4 Sep 2020 21:22:06 +0000 Subject: [PATCH 001/113] Detection metric release. --- argoverse/data_loading/object_label_record.py | 8 +- argoverse/evaluation/detection_utils.py | 135 ++++++++++ argoverse/evaluation/eval_detection.py | 253 ++++++++++++++++++ tests/test_data/detection/1/city_info.json | 1 + tests/test_data/detection/1/lidar/PC_0.ply | Bin 0 -> 369 bytes tests/test_data/detection/1/lidar/PC_1.ply | Bin 0 -> 369 bytes tests/test_data/detection/1/lidar/PC_2.ply | Bin 0 -> 369 bytes tests/test_data/detection/1/lidar/metadata | 11 + .../tracked_object_labels_0.json | 4 + .../tracked_object_labels_1.json | 4 + .../tracked_object_labels_2.json | 4 + .../1/poses/city_SE3_egovehicle_0.json | 1 + .../1/poses/city_SE3_egovehicle_1.json | 1 + .../1/poses/city_SE3_egovehicle_2.json | 1 + .../00000000-0000-0000-0000-000000000000.json | 1 + .../00000000-0000-0000-0000-000000000001.json | 1 + .../detection/1/vehicle_calibration_info.json | 1 + .../tracked_object_labels_0.json | 4 + .../tracked_object_labels_1.json | 4 + .../tracked_object_labels_2.json | 4 + .../detection/test_figures/VEHICLE.png | Bin 0 -> 12330 bytes tests/test_eval_detection.py | 95 +++++++ 22 files changed, 532 insertions(+), 1 deletion(-) create mode 100644 argoverse/evaluation/detection_utils.py create mode 100644 argoverse/evaluation/eval_detection.py create mode 100644 tests/test_data/detection/1/city_info.json create mode 100644 tests/test_data/detection/1/lidar/PC_0.ply create mode 100644 tests/test_data/detection/1/lidar/PC_1.ply create mode 100644 tests/test_data/detection/1/lidar/PC_2.ply create mode 100644 tests/test_data/detection/1/lidar/metadata create mode 100644 tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json create mode 100644 tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json create mode 100644 tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json create mode 100644 tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json create mode 100644 tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json create mode 100644 tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json create mode 100644 tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json create mode 100644 tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json create mode 100644 tests/test_data/detection/1/vehicle_calibration_info.json create mode 100644 tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json create mode 100644 tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json create mode 100644 tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json create mode 100644 tests/test_data/detection/test_figures/VEHICLE.png create mode 100644 tests/test_eval_detection.py diff --git a/argoverse/data_loading/object_label_record.py b/argoverse/data_loading/object_label_record.py index 07a3c69c..ec6eb3ed 100644 --- a/argoverse/data_loading/object_label_record.py +++ b/argoverse/data_loading/object_label_record.py @@ -27,6 +27,7 @@ def __init__( occlusion: int, label_class: Optional[str] = None, track_id: Optional[str] = None, + score: float = 1.0, ) -> None: """Create an ObjectLabelRecord. @@ -48,6 +49,7 @@ def __init__( self.occlusion = occlusion self.label_class = label_class self.track_id = track_id + self.score = score def as_2d_bbox(self) -> np.ndarray: """Construct a 2D bounding box from this label. @@ -271,7 +273,11 @@ def json_label_dict_to_obj_record(label: Dict[str, Any]) -> ObjectLabelRecord: track_id = label["track_label_uuid"] else: track_id = None - obj_rec = ObjectLabelRecord(quaternion, translation, length, width, height, occlusion, label_class, track_id) + if "score" in label: + score = label["score"] + else: + score = 1.0 + obj_rec = ObjectLabelRecord(quaternion, translation, length, width, height, occlusion, label_class, track_id, score) return obj_rec diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py new file mode 100644 index 00000000..abf40da1 --- /dev/null +++ b/argoverse/evaluation/detection_utils.py @@ -0,0 +1,135 @@ +from enum import Enum, auto +from typing import List, Tuple + +import numpy as np +import pandas as pd +from scipy.spatial.distance import cdist +from scipy.spatial.transform import Rotation as R + +from argoverse.data_loading.object_label_record import ObjectLabelRecord +from argoverse.utils.transform import quat2rotmat + + +class SimFnType(Enum): + CENTER = auto() + IOU_2D = auto() + IOU_3D = auto() + + +class DistFnType(Enum): + TRANSLATION = auto() + SCALE = auto() + ORIENTATION = auto() + + +def filter_annos(annos: np.ndarray, target_class: str, range_metric="euclidean", max_dist=50) -> np.ndarray: + """Filter the annotations based on a set of conditions. + + Args: + annos: The annotations to be filtered. + target_class: The name of the class of interest. + range_metric: The range metric used for filtering. + + Returns: + The filtered annotations. + """ + annos = np.array([dt_anno for dt_anno in annos if dt_anno.label_class == target_class]) + + if range_metric == "euclidean": + centers = np.array([dt.translation for dt in annos]) + filtered_annos = np.array([]) + + if centers.shape[0] > 0: + dt_dists = np.linalg.norm(centers, axis=1) + filtered_annos = annos[dt_dists < max_dist] + + return filtered_annos + + +def get_ranks(dt_annos: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get the rankings for the detection annotations. + + Args: + dt_annos: Detection annotations. + + Returns: + scores: The detection scores. + ranks: The ranking for the detections. + + """ + scores = np.array([dt_anno.score for dt_anno in dt_annos]) + ranks = scores.argsort()[::-1] + return np.expand_dims(scores, 1)[ranks], ranks + + +def interp(prec: np.ndarray, method="all") -> np.ndarray: + """Interpolate the precision over all recall levels. + + Args: + prec: Precision at all recall levels. + method: Accumulation method. + + Returns: + Interpolated precision at all recall levels. + """ + if method == "all": + return np.maximum.accumulate(prec[::-1])[::-1] + + +def sim_fn(dt_annos: np.ndarray, gt_annos: np.ndarray, metric: SimFnType) -> np.ndarray: + if metric == SimFnType.CENTER: + dt_centers = np.array([dt.translation for dt in dt_annos]) + gt_centers = np.array([gt.translation for gt in gt_annos]) + sims = -cdist(dt_centers, gt_centers) + elif metric == SimFnType.IOU_2D: + raise NotImplemented("This similarity metric is not implemented!") + elif metric == SimFnType.IOU_3D: + raise NotImplemented("This similarity metric is not implemented!") + else: + raise NotImplemented("This similarity metric is not implemented!") + return sims + + +def get_error_types(match_scores: np.ndarray, thresh: float, metric: SimFnType) -> np.ndarray: + # Euclidean distance represented as a "similarity" metric. + if metric == SimFnType.CENTER: + return match_scores > -thresh + else: + raise NotImplemented("This similarity metric is not implemented!") + + +def dist_fn(dt_df: pd.DataFrame, gt_df: pd.DataFrame, metric: DistFnType) -> np.ndarray: + if metric == DistFnType.TRANSLATION: + dt_centers = np.vstack(dt_df["translation"].array) + gt_centers = np.vstack(gt_df["translation"].array) + trans_errors = np.linalg.norm(dt_centers - gt_centers, axis=1) + return trans_errors + elif metric == DistFnType.SCALE: + dt_dims = dt_df[["width", "length", "height"]] + gt_dims = gt_df[["width", "length", "height"]] + inter = np.minimum(dt_dims, gt_dims).prod(axis=1) + union = np.maximum(dt_dims, gt_dims).prod(axis=1) + scale_errors = 1 - (inter / union) + return scale_errors + elif metric == DistFnType.ORIENTATION: + dt_yaws = R.from_quat(np.vstack(dt_df["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] + gt_yaws = R.from_quat(np.vstack(gt_df["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] + orientation_errors = np.abs((dt_yaws - gt_yaws + np.pi) % (2 * np.pi) - np.pi) + return orientation_errors + else: + raise NotImplemented("This distance metric is not implemented!") + + +def get_label_orientations(labels: List[ObjectLabelRecord]) -> float: + """Get the orientation (yaw) of a label. + + Args: + label: The label + + Returns: + The float orientation (yaw angle) of the label + """ + R = [quat2rotmat(l.quaternion) for l in labels] + v = np.array([1, 0, 0])[:, np.newaxis] + orientations = np.matmul(R, v) + return np.arctan2(orientations[:, 1, 0], orientations[:, 0, 0]) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py new file mode 100644 index 00000000..e75e90e5 --- /dev/null +++ b/argoverse/evaluation/eval_detection.py @@ -0,0 +1,253 @@ +# +import os +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import List, Tuple + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +import click +from argoverse.data_loading.object_label_record import read_label +from argoverse.evaluation.detection_utils import ( + DistFnType, + SimFnType, + dist_fn, + filter_annos, + get_error_types, + get_ranks, + interp, + sim_fn, +) +from tqdm.contrib.concurrent import process_map + + +@dataclass +class DetectionCfg: + """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. + + Args: + sim_ths: The similarity thresholds for determining a true positive. + sim_fn_type: The type of similarity function to be used for calculating average precision. + n_rec_samples: The number of recall points to sample uniformly in [0, 1]. + wdcs: The weight vector for the weighted detection composite score (WDCS). + tp_threshold: The center distance threshold for the true positive metrics. + significant_digits: The precision for metrics. + """ + + sim_ths: Tuple[float] = (0.5, 1.0, 2.0, 4.0) + sim_fn_type: SimFnType = SimFnType.CENTER + n_rec_samples: int = 101 + ads_weighting: Tuple[float] = (3, 1, 1, 1) + tp_thresh: float = 2.0 + significant_digits: int = 3 + + +@dataclass +class DetectionEvaluator: + """Instantiates a DetectionEvaluator object for evaluation. + + Args: + dt_fpath: The path to the folder which contains the detections. + gt_fpath: The path to the folder which contains all the logs. + fig_fpath: The path to the folder which will contain the output figures. + dt_cfg: The detection configuration settings. + """ + + dt_fpath: Path = Path("detections") + gt_fpath: Path = Path("/data/argoverse") + figures_fpath: Path = Path("figs") + dt_cfg: DetectionCfg = DetectionCfg() + + def evaluate(self) -> pd.DataFrame: + """Evaluate detection output and return metrics. + + Returns: + The evaluation metrics. + """ + + dt_fpaths = list(self.dt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + gt_fpaths = list(self.gt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + + assert len(dt_fpaths) == len(gt_fpaths) + data = defaultdict(list) + cls_inst_map = defaultdict(int) + + chunk_size = max(1, len(gt_fpaths) // os.cpu_count()) + accum = process_map(self.accumulate, gt_fpaths, chunksize=chunk_size) + for frame_stats, frame_cls_to_inst in accum: + for cls_name, cls_stats in frame_stats.items(): + data[cls_name].append(cls_stats) + for cls_name, num_inst in frame_cls_to_inst.items(): + cls_inst_map[cls_name] += num_inst + + data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) + + summary = self.summarize(data, cls_inst_map) + summary = ( + pd.DataFrame.from_dict(summary, orient="index", columns=["AP", "ATE", "ASE", "AOE", "WDCS"]) + .round(self.dt_cfg.significant_digits) + .sort_index() + ) + summary.index = summary.index.str.capitalize() + + summary.loc[""] = ["", "", "", "", ""] + summary.loc["Means"] = summary.iloc[:-1].mean().round(self.dt_cfg.significant_digits) + return summary + + def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: + """Accumulate the statistics for each LiDAR frame. + + Args: + gt_fpath: The ground truth file path. + + Returns: + The aggregated data used for summarization. + + """ + log_id = gt_fpath.parents[1].stem + ts = gt_fpath.stem.split("_")[-1] + dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/tracked_object_labels_{ts}.json" + + dt_annos = np.array(read_label(dt_fpath)) + gt_annos = np.array(read_label(gt_fpath)) + gt_clss = np.array([gt_anno.label_class for gt_anno in gt_annos]) + + class_data = defaultdict(list) + class_to_ninst = defaultdict(int) + for gt_cls in np.unique(gt_clss): + dt_filtered = filter_annos(dt_annos, gt_cls) + gt_filtered = filter_annos(gt_annos, gt_cls) + + if dt_filtered.shape[0] > 0: + error_types = self.assign(dt_filtered, gt_filtered) + class_data[gt_cls] = error_types + + class_to_ninst[gt_cls] = gt_filtered.shape[0] + return class_data, class_to_ninst + + def assign(self, dt_annos: np.ndarray, gt_annos: np.ndarray) -> List: + """Attempt assignment of each detection to a ground truth annotation. + + Args: + dt_annos: Detection annotations. + gt_annos: Ground truth annotations. + + Returns: + True positives, false positives, scores, and translation errors. + + """ + n_threshs = len(self.dt_cfg.sim_ths) + error_types = np.zeros((dt_annos.shape[0], n_threshs + 3)) + scores, ranks = get_ranks(dt_annos) + if gt_annos.shape[0] == 0: + return np.hstack((error_types, scores)) + + match_matrix = sim_fn(dt_annos, gt_annos, self.dt_cfg.sim_fn_type) + + # Get the most similar GT annotation to each detection. + gt_matches = np.expand_dims(match_matrix[ranks].argmax(axis=1), axis=0) + + # Grab the corresponding similarity score for each assignment. + match_scores = np.take_along_axis(match_matrix[ranks].T, gt_matches, axis=0).squeeze(0) + + # Find the indices of the "first" detection assigned to each GT annotation. + unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) + for i, thresh in enumerate(self.dt_cfg.sim_ths): + tp_mask = get_error_types(match_scores[unique_dt_matches], thresh, self.dt_cfg.sim_fn_type) + error_types[unique_dt_matches, i] = tp_mask + + if thresh == self.dt_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: + dt_tp_indices = unique_dt_matches[tp_mask] + gt_tp_indices = unique_gt_matches[tp_mask] + + dt_df = pd.DataFrame([dt.__dict__ for dt in dt_annos[ranks][dt_tp_indices]]) + gt_df = pd.DataFrame([gt.__dict__ for gt in gt_annos[gt_tp_indices]]) + + trans_error = dist_fn(dt_df, gt_df, DistFnType.TRANSLATION) + scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) + orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) + + error_types[dt_tp_indices, n_threshs : n_threshs + 3] = np.vstack( + (trans_error, scale_error, orient_error) + ).T + + return np.hstack((error_types, scores)) + + def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict: + """Calculate and print the detection metrics. + + Args: + data: The aggregated data used for summarization. + cls_inst_map: Map of classes to number of instances. + + Returns: + summary: The summary statistics. + + """ + summary = defaultdict(list) + rec_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) + num_ths = len(self.dt_cfg.sim_ths) + if not self.figures_fpath.is_dir(): + self.figures_fpath.mkdir(parents=True, exist_ok=True) + for cls_name, cls_stats in data.items(): + num_inst = cls_inst_map[cls_name] + ranks = cls_stats[:, -1].argsort()[::-1] + cls_stats = cls_stats[ranks] + for i, _ in enumerate(self.dt_cfg.sim_ths): + tp = cls_stats[:, i].astype(bool) + + cumulative_tp = np.cumsum(tp, dtype=np.int) + cumulative_fp = np.cumsum(~tp, dtype=np.int) + cumulative_fn = num_inst - cumulative_tp + + precs = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) + recs = cumulative_tp / (cumulative_tp + cumulative_fn) + + precs = interp(precs) + prec_interp = np.interp(rec_interp, recs, precs, right=0) + ap_th = prec_interp.mean() + summary[cls_name] += [ap_th] + + # AP Metric + ap = np.array(summary[cls_name][:num_ths]).mean() + + # TP Error Metrics + tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + 3], axis=0) + + ads_summands = np.hstack((ap, 1 - tp_metrics)) + + # Ranking metric + wdcs = np.average(ads_summands, weights=self.dt_cfg.ads_weighting) + + summary[cls_name] = [ap, *tp_metrics, wdcs] + self.plot(rec_interp, prec_interp, cls_name) + return summary + + def plot(self, rec_interp, prec_interp, cls_name) -> None: + plt.plot(rec_interp, prec_interp) + plt.title("PR Curve") + plt.xlabel("Recall") + plt.ylabel("Precision") + plt.savefig(f"{self.figures_fpath}/{cls_name}.png") + plt.close() + + +@click.command() +@click.option("--dt_fpath", "-d", help="Detection root folder path.", type=click.Path(exists=True), required=True) +@click.option("--gt_fpath", "-g", help="Ground truth root folder path.", type=click.Path(exists=True), required=True) +@click.option("--fig_fpath", "-f", help="Figures root folder path.", type=str, default="figs/") +def main(dt_fpath: str, gt_fpath: str, fig_fpath: str) -> None: + dt_fpath = Path(dt_fpath) + gt_fpath = Path(gt_fpath) + fig_fpath = Path(fig_fpath) + + evaluator = DetectionEvaluator(dt_fpath, gt_fpath) + metrics = evaluator.evaluate() + print(metrics) + + +if __name__ == "__main__": + main() diff --git a/tests/test_data/detection/1/city_info.json b/tests/test_data/detection/1/city_info.json new file mode 100644 index 00000000..ec634ac4 --- /dev/null +++ b/tests/test_data/detection/1/city_info.json @@ -0,0 +1 @@ +{"city_name": "PIT"} diff --git a/tests/test_data/detection/1/lidar/PC_0.ply b/tests/test_data/detection/1/lidar/PC_0.ply new file mode 100644 index 0000000000000000000000000000000000000000..762fa74ee2e929940408edcf1e9d3e2d0492b770 GIT binary patch literal 369 zcmZXOK@Ng25JkIY)U11At};0>^ZAvP(6lxnCej>1tmhD(+%UApuSMnmu?KQr?> zzin$mR5h+vUKZLa7jmP$HUDX-tI};8%uv_~3=1KY poLw0)8Y+CK{n@4zFF@b&t6KsgH>^tV&icMfLdjnT#T_XSh literal 0 HcmV?d00001 diff --git a/tests/test_data/detection/1/lidar/PC_1.ply b/tests/test_data/detection/1/lidar/PC_1.ply new file mode 100644 index 0000000000000000000000000000000000000000..762fa74ee2e929940408edcf1e9d3e2d0492b770 GIT binary patch literal 369 zcmZXOK@Ng25JkIY)U11At};0>^ZAvP(6lxnCej>1tmhD(+%UApuSMnmu?KQr?> zzin$mR5h+vUKZLa7jmP$HUDX-tI};8%uv_~3=1KY poLw0)8Y+CK{n@4zFF@b&t6KsgH>^tV&icMfLdjnT#T_XSh literal 0 HcmV?d00001 diff --git a/tests/test_data/detection/1/lidar/PC_2.ply b/tests/test_data/detection/1/lidar/PC_2.ply new file mode 100644 index 0000000000000000000000000000000000000000..762fa74ee2e929940408edcf1e9d3e2d0492b770 GIT binary patch literal 369 zcmZXOK@Ng25JkIY)U11At};0>^ZAvP(6lxnCej>1tmhD(+%UApuSMnmu?KQr?> zzin$mR5h+vUKZLa7jmP$HUDX-tI};8%uv_~3=1KY poLw0)8Y+CK{n@4zFF@b&t6KsgH>^tV&icMfLdjnT#T_XSh literal 0 HcmV?d00001 diff --git a/tests/test_data/detection/1/lidar/metadata b/tests/test_data/detection/1/lidar/metadata new file mode 100644 index 00000000..f05822b3 --- /dev/null +++ b/tests/test_data/detection/1/lidar/metadata @@ -0,0 +1,11 @@ +x y z intensity laser_number +0 0.0 0.0 5.0 4.0 31.0 +1 1.0 0.0 5.0 1.0 14.0 +2 2.0 0.0 5.0 0.0 16.0 +3 3.0 0.0 5.0 20.0 30.0 +4 4.0 0.0 5.0 3.0 29.0 +5 5.0 0.0 5.0 1.0 11.0 +6 6.0 0.0 5.0 31.0 13.0 +7 7.0 0.0 5.0 2.0 28.0 +8 8.0 0.0 5.0 5.0 27.0 +9 9.0 0.0 5.0 6.0 10.0 diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json new file mode 100644 index 00000000..39f3c80a --- /dev/null +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE"}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE"} +] diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json new file mode 100644 index 00000000..11f63d81 --- /dev/null +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE"}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE"} +] diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json new file mode 100644 index 00000000..9b27b559 --- /dev/null +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE"}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE"} +] diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json new file mode 100644 index 00000000..c8562dc8 --- /dev/null +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json @@ -0,0 +1 @@ +{"rotation": [0, 1, 0, 0], "translation": [1, 0, 0]} diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json new file mode 100644 index 00000000..07a39746 --- /dev/null +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json @@ -0,0 +1 @@ +{"rotation": [1, 0, 0, 0], "translation": [2, 0, 0]} diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json new file mode 100644 index 00000000..c17eb32b --- /dev/null +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json @@ -0,0 +1 @@ +{"rotation": [0, 0, 1, 0], "translation": [2, 1, 0]} diff --git a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json new file mode 100644 index 00000000..4fbfa530 --- /dev/null +++ b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json @@ -0,0 +1 @@ +{"label_class": "VEHICLE", "uuid": "00000000-0000-0000-0000-000000000000", "log_id": "1", "track_label_frames": [{"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE"}, {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE"}, {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE"}]} diff --git a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json new file mode 100644 index 00000000..66fec672 --- /dev/null +++ b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json @@ -0,0 +1 @@ +{"label_class": "VEHICLE", "uuid": "00000000-0000-0000-0000-000000000001", "log_id": "1", "track_label_frames": [{"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE"}, {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE"}, {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE"}]} diff --git a/tests/test_data/detection/1/vehicle_calibration_info.json b/tests/test_data/detection/1/vehicle_calibration_info.json new file mode 100644 index 00000000..83c8a128 --- /dev/null +++ b/tests/test_data/detection/1/vehicle_calibration_info.json @@ -0,0 +1 @@ +{"camera_data_": [{"key": "image_raw_stereo_front_right", "value": {"focal_length_x_px_": 3634.1664215146284, "focal_length_y_px_": 3634.1664215146284, "focal_center_x_px_": 1260.6433779229542, "focal_center_y_px_": 1059.5205903709868, "skew_": 0.0, "distortion_coefficients_": [-0.08785433857565009, -0.2963843841153259, 2.3674681480992428], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.5007328527621485, -0.495680186520672, 0.5013368080254209, -0.5022242206170416]}, "translation": [1.6421058178669956, -0.1499718687167596, 1.361891429230726]}}}, {"key": "image_raw_ring_rear_left", "value": {"focal_length_x_px_": 1393.318962096269, "focal_length_y_px_": 1393.318962096269, "focal_center_x_px_": 967.2847108481072, "focal_center_y_px_": 613.2332689161747, "skew_": 0.0, "distortion_coefficients_": [-0.17241307888830998, 0.11907008773383419, -0.027945198361512744], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.608056280997397, -0.6173801180944725, -0.3514622121439918, 0.35437785251920234]}, "translation": [1.105844707913899, 0.24650725363322967, 1.3735906821517418]}}}, {"key": "image_raw_stereo_front_left", "value": {"focal_length_x_px_": 3660.425411174431, "focal_length_y_px_": 3660.425411174431, "focal_center_x_px_": 1246.9202677240673, "focal_center_y_px_": 1063.4338217897928, "skew_": 0.0, "distortion_coefficients_": [-0.09737880168798231, 0.025226817408578506, 0.6478859260574071], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.49725711847079235, -0.5033139293759432, 0.4991231316939582, -0.5002864639726806]}, "translation": [1.619756836560706, 0.14842013734897835, 1.3648467699127986]}}}, {"key": "image_raw_ring_rear_right", "value": {"focal_length_x_px_": 1393.4158534327278, "focal_length_y_px_": 1393.4158534327278, "focal_center_x_px_": 951.6911354519806, "focal_center_y_px_": 597.3719168271283, "skew_": 0.0, "distortion_coefficients_": [-0.171660451822291, 0.11750204460346327, -0.027417995114452046], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.35852818774144446, -0.3519145089444871, -0.6125134020323175, 0.6102794845970104]}, "translation": [1.0985731208497924, -0.24534171331957055, 1.37160293757265]}}}, {"key": "image_raw_ring_side_left", "value": {"focal_length_x_px_": 1395.0857466752807, "focal_length_y_px_": 1395.0857466752807, "focal_center_x_px_": 967.7390137760952, "focal_center_y_px_": 613.3380881206222, "skew_": 0.0, "distortion_coefficients_": [-0.17228684997200724, 0.11667402061297209, -0.02528530850263856], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.7004567328555497, -0.7088187475676434, -0.05809163085288814, 0.05968007137075483]}, "translation": [1.2934806248356339, 0.27738067568824815, 1.371113292276382]}}}, {"key": "image_raw_ring_side_right", "value": {"focal_length_x_px_": 1391.654458212649, "focal_length_y_px_": 1391.654458212649, "focal_center_x_px_": 981.9614016400475, "focal_center_y_px_": 608.2719144486357, "skew_": 0.0, "distortion_coefficients_": [-0.1723517502956347, 0.11731566692457349, -0.02574725269922101], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.06401399257908719, -0.06266155729362148, -0.7078861012523953, 0.7006232979606847]}, "translation": [1.294530313917792, -0.28519924870913804, 1.3701008006525792]}}}, {"key": "image_raw_ring_front_right", "value": {"focal_length_x_px_": 1392.4445795594495, "focal_length_y_px_": 1392.4445795594495, "focal_center_x_px_": 958.0278877584801, "focal_center_y_px_": 608.506324610186, "skew_": 0.0, "distortion_coefficients_": [-0.1723521614846795, 0.12188989312717174, -0.03271981337763657], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.2653173446516557, -0.2685570788918094, 0.6608505371335351, -0.6486604424307151]}, "translation": [1.507413796878202, -0.2667563199332462, 1.3615052881313203]}}}, {"key": "image_raw_ring_front_center", "value": {"focal_length_x_px_": 1392.1069298937407, "focal_length_y_px_": 1392.1069298937407, "focal_center_x_px_": 980.1759848618066, "focal_center_y_px_": 604.3534182680304, "skew_": 0.0, "distortion_coefficients_": [-0.1720396447593493, 0.11689572230654095, -0.02511932396889168], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.49605542988442836, -0.49896196582115804, 0.5027901707576079, -0.5021633313331392]}, "translation": [1.6519358245144808, -0.0005354981581146487, 1.3613890006792675]}}}, {"key": "image_raw_ring_front_left", "value": {"focal_length_x_px_": 1393.7674787830663, "focal_length_y_px_": 1393.7674787830663, "focal_center_x_px_": 960.5401682856536, "focal_center_y_px_": 607.4003449968101, "skew_": 0.0, "distortion_coefficients_": [-0.17299777690816825, 0.1226893513068073, -0.03042136313516928], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.6480281666752452, -0.6560752002901659, 0.2739859090547071, -0.27305045028200403]}, "translation": [1.511797877348265, 0.25337046635825855, 1.3651560815686514]}}}], "vehicle_SE3_up_lidar_": {"rotation": {"coefficients": [0.9998848409086823, 0.0011389828365580806, -0.0010769852622658116, -0.015094626257773826]}, "translation": [1.3504724404154866, -0.003966310091761113, 1.5700506059693788]}, "vehicle_SE3_down_lidar_": {"rotation": {"coefficients": [5.6898203165648547e-05, -0.994250135333888, -0.10708208174631563, -0.0003048278454676608]}, "translation": [1.3502875642048289, -0.0019957763317242708, 1.4771699630675978]}} diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json new file mode 100644 index 00000000..28af00f7 --- /dev/null +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE", "score": 1.0}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE", "score": 1.0} +] diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json new file mode 100644 index 00000000..b15b4a1f --- /dev/null +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE", "score": 1.0}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE", "score": 1.0} +] diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json new file mode 100644 index 00000000..6bcb8d22 --- /dev/null +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -0,0 +1,4 @@ +[ + {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE", "score": 1.0}, + {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE", "score": 1.0} +] diff --git a/tests/test_data/detection/test_figures/VEHICLE.png b/tests/test_data/detection/test_figures/VEHICLE.png new file mode 100644 index 0000000000000000000000000000000000000000..3c2c83ceb8367bd98c9505cf1f542c8e6f6670c7 GIT binary patch literal 12330 zcmdsdcU;f=`}ezWkVD5vq+#Tsl96ar%l7LTUD{IKSI|4_5kJ`(V_KI|?c_A*K-1wJ zEP6jX?V77wQqb#gbvVl@*EO5XPhXv$qTCPkDB02BQHFTvx8kMV3U|D?wT;4On{H7n zDC+#C|AY@hPoI&J^2u+t$a8jaVOqEDxnBOQ?yt{I47hGNd}ViONkM^xbig6`bLZ|U zC@OOEcU0@T#o{Y_L`1ZB{m(M%IwuO*4a6|(*dHtK=7{gnUh2-0pPsdfa$ig{EZwf{ z(C#bbGLv%HwC=?pe=Id^$dDJd?UT!K9B-I^CH?eOu&7g1Xz0=Ubo1&*dlW0eBp6@5 zeAzuxv~RKT>bmvo`Hd@fry7-?ZY%IsrP=l?$;pLQR49FUdPMfqr%xK0mM?>73{)jf zBlNUUb8gP|9XobPiaJdw$;&@@B@+~-9IayXHT2kktG}*mW?bHaqol&*Kx1X-u_dcF zA3SNEGw%25+!M>D13nKQ@>_mC)95_{iZmA0HxzNYw^GETADHu(D5lFoiR-a`vx<+6FSFGl6BKDAnOc^zrmy>_8EQZqen zX1ur4bf z>Xm8LNnZ5E=h>#eeDx|yH_z3$BH{t|T*#*9h1-0uZXQjSl9OM0l;W>z%5mB$iOUF@ z=ef*INczNOe7iF`!Og{0wkyQ>`R&^)Zr-}3BK;um>f-!lQMYxE+v22K>Xq7qjkfis z#&s!?8~17`MVTuo_>iMq!zOgbM{Isf@zkkP-S0ft2OKdEUYH+r>kTl8D|M$^^61f{ zOJ{Sa&-r`KQ~XlcX@*s+#<5Q0whvy89Zw9D@?7WFaa;N*X=%m0eS3V8-`!hmCZaCY zBpeUBm|b5|Sg4?*6Z|$m-?ZN2sacj}JPyV)&1o=4)vB{t+156Ve#w%Zk~nS!DXBX; zF5}8}dGcYWyQ8Ax$m49+ZBAZ1iDT4MS66qO8!oVEcA1QnVOtnn)8gH#yO!cF9Iw{h z7hWETo$syEG*?c({AH}wgM(}L?ue6lbW48uh#h!-msdea>5-+S<=C6$91+KE{8Fhj zy_h*aQGYhI(QR>_#H=5$c8{56is1vqgLR4MbbULze(&3lUWAB30SKqmFhpTdMx-+C2aTw{f9QU%3k;Mo@T`uI5918dezVkF{0@o8M`@<@S>{~=Ayu)OwAn_B{M93q zlhruH@t){-eg5E&MmLq07l6Snlt92J=st=}@B88h{nJHIBZgHh{cbUG#PJyvKbpx#st@K&2G6>vsi}F=tao-lloc!e z^l%rqavL^wuYb|Y7cV$>?%Wx9>h8un_wU=fHNBGFqU}1}K7;jd*t}W2BieO-#+PM1 zRrr=+tGmXVZfnG^QC4ppZLE+x-tgNy&-RJ_y42{7;-F{*&7SSsk7Mx_%BtG!xzD~M zE#+n>`kB_R-yz8&VxJJ~JR&JNUZL1i8s=d;&=6UhY(O*bh=}-QHA`1oy_q61LMBk) z+10iWzFE)O5LT)oVlEXlJM-0SLP~ozQoati6{MO}^P1MDNe|}Enzip*PPyJ-I!dK} z2(oL6Ib1Yct zw+JAJG;hxBY09CcnK#E3aJXrCi;UbxROv(wUR$#KkXeI(&0tebRM9UK74g71n&y6| zt32F$W~^JWbz!vQi(--A-f1!U8#zBa*&VjYzmvGqf%~eZq|37vt#|~&Bluh^LIQwue#}IY4J#ojE`3i zG}3IkU;6vinK?UWfBW{WVbIFba>X^oXm3R!Rg-F>irYEutg8ZofBbkxFi3{*oVvt| z5}qZyWC9}^8nkjo3c1^F35kn~W9hXjN3iN$IA0NmkrxvKjlHgU=FMWDI*}++#&6s8m*Si12FROCNtj3GGHjDE#=*!i}AsJ&xa`%CN5{(Xl_ZnuOMu z0RHHAJI*6>&$TBMH?m!mD*E(E9y^wloXqswZ*q<8SS~}$l=j_0dPK+3+W%1Q`)|VG zv$s=bbt#eu4jeFU%Fbk6 zaclLKDAjl!DM`sh6J{Kx@vreK+{0yp6oBIVP7{3$iP26q1ym>UZkq69Kcg7A4*7zsj}Eixwbw1t|VzOvm~-&PBO6w7w0_a8p^)psTq zu$Kd32-(I_8tWs5XkDy@Y=fMq3Op(w90isT61GK?hhDwm?u-C7T>KhnvZ`~Sd zDGc2dwXDE=pE_Mk%=hn?PDx1k8}>HR(jhC9<>cf#>_$dL2*E$?#o|kAoSSoQ|Ni}v zJuu7usm5lCkB<*^MCtk7-Mi1+ym_;*u#jt1c>V9^Q%B{j9jC~Ce>o!4vK+YZ$LwfA!hSXZi_nV0yQBkNfSIXA@S5#dPQ9MB zn49pS)D(xAp?nAPhwIPE(2+3d`PXR0zdu}0)nyJVhRbwkb_SONEMvkXyrVKQGTwD| zcX!*aVMIYZSi?ebI1e2oB9;V*(isI;X0@RgcS5Z#YSJV?=KgOXs4q)VOwhm0I zpSrUSymayeP)b@oNeo|+~7{rZ%E)5Mkbjt(v!o^$>>j*+~yzJoVCJh+b>QH{@? zi3&PFB9S$%X&EaEOBBesQDc_P464v2bw8pzb#feQ1$nU4;v374Ez)V7WT4z9Tc=F( z8Cqi0lgGptN5w5OwE_T=4znY>$&r*qk1)L+8nS@$Cb=(fZ*M=nM3_}dRyMxyndtq+ zV-(fmf~`)V&j80dxQNqs4|r2Wb9al5h&W?pWHi(6uPgH7o(p$l=`21!KA@3s{@RP= z*1=m_%;Y+YgFsE9R&P06BX|h6E7UWgw9>9T5wh*uE$Ik_l;XDN>N44A$HI5{UanX8p5K2z4Y2UfZ~Y1b z#C1{_qD|@K$vP__J+Y_N#>zP!1l>o;!r`um5&x%pU9B1@{;g$vJtAKW?8Ai2KWs$7Q;r!|Oq zi%vXNiPwqMrrBoJpT4*084kZtw1apOz#2yJxv!0#``o7Q5_EijmsvGpBh73_I?vhg zRd=GUH~~nw-NL9qS*67d~G%`B+kaK!!%JA(UE7QE5 z;t{&Ky4IQII89bvU$zR^v=y!bpv#L*I87;pcuEcZ9i5%r;^NKoV!L*Q8?RbilwsZw z*{Hafs9%r!Jq(hatU6_5>cxFJ$>sE5M%#S_(-XITO|<#^`SZDD%a-{Xu3}(N(9?UC zl$5lAl~v0u1RwlFvOS8gY7~BQ-MWzLv;CICz_7(e9hH#Ly?g})1>C&6#pI?$Rw>Jy ztz3}d=y%St^EC$*FmD@-nG#(Rk-^32zAtCY2U$*tBgvcw*EJsbX;cm3UV+;|!kgVU zK0cl|^SjUJ#B_gDi`(evs8z^^w1@ev=QE)zYF?=_fHS&Xm;_l^SLgwfVNjCev$e`e#A8Z1dYKT zkrHw$7z8viF*An~Sq!}ZXxn|&ZV;u(CV%yjE_fxXz^d^iBBEy_zDqZU~#; zOTAVUHgb(9Wb@0qaM_SJ;lb?WxAl_J(#jw|6_JX&Bo(x@0*Ol&9_%RPWBvW(TK!9x zGG;wIJWP-@1pnePJ6NQHgora#1_3ZIP#&v!WUqSCNstKzk2P!rF5$*So%QEs&Z3cM zeN)v*9fo`faUSuE%bn?2*H3KBc%2;Be&V&Lazm`!pk^5h%%7@K2Z5!dw=U&y%6AVUi=IQAh$+yNoY|!*XXqpn%$tv!{4R*{`pvg&AQ;BDNur(l50FpY~H*%V87mR z&*pXzE8LZZ(sdp_H9t36LflH35V2TxQOB;NB3{M$nep+y;1KJsWd34N z_CgZ=Pubq|hUZ|>KY|}Zu74A0$@pF>VrT((O*#LDel@FroDUJi^76p{!;UmNC;zDb z>gXgrz?;WUoZz?ZQ=6aZjkoQ`HXQXLGD7KH*~_#1mp^a%@`O)9US8f2R&ZswEC&JC zy_#t}m2LwKnJ9~@06Sv!!uvFw8frDJNq7beh*Q!$Z$9n!bB{xBu3{!7wl3Mg`_>vZ zE=fmN6C#WF$!s&Tcs3urFw&&d_E>fbrx2#(k5W?5TXS*a= zQ#Htybmj(LzzCJ;N{DgpEP$v4|bG<#KRi38E8<)Q&+yd zxeD?yG$w{;+4AKkEw8W50361{#pfmc^dl1zzviZf?E;K2nmM-Kj3!Lra*%P-F|&%^!; zfagp&|NLmkqP006$R(_NB?7hhLVqJoRaR{J1!;1LNrMH;ZE-k7vF<WJb$|h}C6Ug=?f;QIA?|$h2bO;E0EpOT?tqov8d* zC_v6+7C=(66n7&qTJHW8sS^L>@E+d)gu_g;8vT~CY|y*WZonq9U`WU z)?!egOy{yn-@lq!E!Wdf6h-c~r85W8i&=HtOEYVXB2u9rw)YHJO3@sYRjwAVs^?+U z1l08iG`@UN9LJ4q`sT`k|XViG4?;375*c2e{mW$hPp%DYGmp}q82{GvNVrc^wFV)3 z)BXN87mbDE|56N1y1qX1;Y3xDRJeLIG2pOi6qNZ1bgYP702%7JF8qlb_qJ`vKoSR3 z4Gj$|`&eeW3V=a?M!s%z7g0Ez}TcZGG)zP(D4C#S8Y|8diVfaPO`}^$uqeZtH7fXF zQA2P@NC-VWy;L?rBU>7^)dzj9@afS`lh(I4VH5a&xX%`S@7}Kwlu?iBiwH&G?)u4pkW$4XvY+3-jW7jtCK@Kc_ zm&1E2YWc65LN6BF@9qu{sd}(Of*gCSdh%)TMjuW#LCcdMy8{b2y?_H3Z`lqu?S;mK zmESX3BCd)ZqsbZ?Mvy)OJdr4HedE&SH;GqL!0z<9oJO!nUT2u~b-{L9Y$aO?nm_7i>NNnl{0aU2jrsUov&BJ*b_@~D|v9OoVw#sS>O zXkTCSIA6bh-NPHEA;9ynZIFY^Fu;`}O=qxq<*fd|Kvv9B^pJMviWL!MrfrM+OM>5s zIQ=r(xA}9jL5*OEpFUdOswk1FK|+^xRu?d|aL9$ru-*LQj~b!VM~_mAukERyU;g>a zTiT;ndJ3BbFMrU%vE5kvqf6VyhiQ>d=-(E0w6%qaFHVgm6fT*OL7Q}z0Ur;|JqRu_TU{4tH;_^G>gtpZe{==$j!b-NmFbu}ut(T3!s159s za+nOHI5|dW3@X<;xX$*cSE4JXZC!c_d00i-*nKqz7?pd2$UxBJ$D4?Pg5ZNDnrR%g z;E6fDQN#@D*sr`h1>--lLBe(#Fk;itaxfsa^ydY*v1Cx15~ z(r1rrm{qckgQ;OE3TCQ;=@F8a++t!)mO?>b#q}aunRbzVA|Y1L!_lBtDj>0BeOOQq z@1@|`@m>R)?y~o=$%0bRpXGKwvUfdrQd~1xCCM#t*0XLBI)g^I67vfI*eZK z#aK!b@I(}sM%TcLY=&Bq|Gsoib!cZ1bGQ39mG4q&tK(<`LME{Ech<#ak4TxoVbY&i zED&GNBb~#Ybqf>edFmLg$}#1y+UOh9n*`6lIN+eZl(N4{v{si0jTM8$J(`@n^wiBtDHG=8y7pww9=h*RrJPichSDj zPx$JxY@*TTIh!IOA<>Hxvu{)w&7YSuWS$1n0OUO(R>jD!&C zlJV6+2cs~qQ?1J}X!L1doypBrPdY23Rk;ryRM?k2bX_CG@ELbp#&Jvml><*j<&#vi z{&X0qihb%jH!MLao}WHSplcbRnn(fr=a(+xz%@iP>hocLDJdzv2n`Kg*AD}nv|dJq z3G_|uqpP24f7Dv6y0@9n8a^bk6Kx-nh?#jJ@s@TVV=n zAhVO5G{jZ4uHFnM=P^2&nUSG};3hhb9RGNKy(;0(3JU9@-Sff@j%HhR%7~79+~L?0 z8J=R(tKx!jvpV$bz{guX6};XiCiD#HnlqZfA#iqfmYSCx0n)r&FtCz=aaGti*xMU@ zmN)ZVnnq6J&Clb4U^$gb7)3(unq5@kNF*hni&w36D8?BY8%L2D01pmVwX)jK3+jFt z^-Hzw*J!kF^J0%|e5&=axwEC^e5zsTaRTjWgw`eTkm;bTxsf8%*xD`HA4PU?a_)wb z#3-IWCwTcWxJ8NxoAJ+lB?$+WPYG;P*{ec{#$apj+`j!4{+Coryk@!#u$OyO7?$nc zN;gdA+_`t}5t937knIi9p57v49ns{_$rjfd>0VfDb@ZsaZC}l^l$1l0BwdWlUwOf& zAzDM$cR-o<`=AxZw$^Zn#lXAGsz;YR3{Kx}PR@5#jyaV#KU-n448h?bFb-8W=Ekiw z)rN&78fQ(vGwftPkO1ez1nzs);{z9C;0dYEP7bE?6eU4spI^-`Qq#ED5IYF4PQ^5k z(O$I#cg<$*h$O4|V&OqGctcg-yqAI5@8*TBemiN?e_b`APh@eU9i8eq{Z7FnN2)Oc zlQ;l(vbWwmFBJmOkjySs-Cz_`)26jBAw^*@CSmhq1+6;Nkij=n^@}zlgUYf7vU;OT zn6r-qkWyV{ZArh@kJALTy9)jOM(_GoHN$=M=lLDb!H+?ESRK_X71MfG(2OdbuZ1TQ z154~tS01{4(P(24$`gxTo(U`l_P81+<;!tSa0n&2##s(ty4|^I@*wK@eDex-CWJ^6 z$PY4kUr}>&GkF5E9AjWsF{=KkdWf|o&?5twjhb)Xyr~L3b|Z$vZLTz5*~uTK-B|Ww zlO<1ze}0l$2t305E2}B!&3}DDR_;A!BF7*!Y0bhvCSIagxar+}U>_O#LXctz@+1DM zu2u!ft$-Caj^R2bJ0nW&BGgiaklS>TF0pv~oAY!>J4>qJONArXmF?93{teROK*!|x z@#CZ^MVb-_!+{d!y&5k_oVQzcV{)t_P|!-sc@d4j3QTlXM#%5LhRC8@OF9;ulH{mn zCI{ou8qszhexEf_XF}Rt-UA%eoP2JjHk|T3C^;N1V-Fx^IrkhmAP)#FgU4M7L)x@C zR~syXB;_FtX9S5jD2O_ak*t4}jk1m)%==6O$jdVH`$$7TRF(cDTH1p_UN zg$1tC8ApHgCKy765GE;ed2^>OFflQ812%j)zXS>@`W!-Q@yp`_C&<{?PRZ8RR)UZ^ zZu2%|+;uQ}^uFuDw1LZXyFVEv)hVW&u>Y3T*#kdf+1$B&hRgj_sAD=eMym2Gf{{3=3#P&ziF*Co} z9IZzw#aHJ_VHd{}CWT51(_RIxgT%yYDvBO6ld+2NI--+NjWAU!k_}{0MBnEe_ka92 z0`qf^rtW)tdvktjZpP#&ySQui0=A^ +"""Detection evaluation unit tests""" + +import logging +import pathlib + +import numpy as np +import pytest +from pandas.core.frame import DataFrame +from scipy.spatial.transform import Rotation as R + +from argoverse.data_loading.object_label_record import ObjectLabelRecord +from argoverse.evaluation.detection_utils import DistFnType, SimFnType, dist_fn, sim_fn +from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator + +TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection" +logging.getLogger("matplotlib.font_manager").disabled = True + + +@pytest.fixture +def evaluator() -> DetectionEvaluator: + detection_cfg = DetectionCfg(significant_digits=32) + return DetectionEvaluator( + TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg + ) + + +@pytest.fixture +def metrics(evaluator: DetectionEvaluator) -> DataFrame: + return evaluator.evaluate() + + +def test_center_similarity() -> None: + olr1 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) + olr2 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) + assert sim_fn(olr1, olr2, SimFnType.CENTER) == -5 + + +def test_iou_2d_similarity() -> None: + # TO DO + pass + + +def test_iou_3d_similarity() -> None: + # TO DO + pass + + +def test_translation_distance() -> None: + df1 = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) + df2 = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) + assert dist_fn(df1, df2, DistFnType.TRANSLATION) == 75 ** (1 / 2) + + +def test_scale_distance() -> None: + df1 = DataFrame([{"width": 5, "height": 5, "length": 5}]) + df2 = DataFrame([{"width": 10, "height": 10, "length": 10}]) + assert (dist_fn(df1, df2, DistFnType.SCALE) == 1 - 0.125).all() + + +def test_orientation_distance() -> None: + # check all of the 45 degree angles + vecs_45_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 4)] + for i in range(len(vecs_45_apart) - 1): + df1 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_45_apart[i])}]) + df2 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_45_apart[i + 1])}]) + assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 4) + assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 4) + # check all of the 90 degree angles + vecs_90_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 2)] + for i in range(len(vecs_90_apart) - 1): + df1 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_90_apart[i])}]) + df2 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_90_apart[i + 1])}]) + assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 2) + assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 2) + + +def test_ap(metrics: DataFrame) -> None: + assert metrics.AP.Means == 1 + + +def test_translation_error(metrics: DataFrame) -> None: + assert metrics.ATE.Means == 0 + + +def test_scale_error(metrics: DataFrame) -> None: + assert metrics.ASE.Means == 0 + + +def test_orientation_error(metrics: DataFrame) -> None: + assert metrics.AOE.Means == 0 + + +def _rotvec_to_quat(rotvec: R) -> R: + return R.from_rotvec(rotvec).as_quat()[[3, 0, 1, 2]] From 357e789eee0c799c17c91ed871a3830ef94b2511 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Mon, 7 Sep 2020 20:57:28 -0400 Subject: [PATCH 002/113] Documentation and function parameter changes. --- argoverse/evaluation/detection_utils.py | 77 ++++++++++++++++--------- argoverse/evaluation/eval_detection.py | 52 ++++++++--------- tests/test_eval_detection.py | 4 +- 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index abf40da1..55b3caab 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -1,3 +1,10 @@ +# +"""Detection utilities for the Argoverse detection leaderboard. + +Accepts detections (in Argoverse ground truth format) and ground truth labels + +""" + from enum import Enum, auto from typing import List, Tuple @@ -12,8 +19,6 @@ class SimFnType(Enum): CENTER = auto() - IOU_2D = auto() - IOU_3D = auto() class DistFnType(Enum): @@ -22,47 +27,53 @@ class DistFnType(Enum): ORIENTATION = auto() -def filter_annos(annos: np.ndarray, target_class: str, range_metric="euclidean", max_dist=50) -> np.ndarray: +class InterpType(Enum): + ALL = auto() + + +def filter_instances( + instances: np.ndarray, target_class: str, range_metric: str = "euclidean", max_dist: float = 50.0 +) -> np.ndarray: """Filter the annotations based on a set of conditions. Args: - annos: The annotations to be filtered. + annos: The instances to be filtered. target_class: The name of the class of interest. range_metric: The range metric used for filtering. Returns: The filtered annotations. """ - annos = np.array([dt_anno for dt_anno in annos if dt_anno.label_class == target_class]) + instances = np.array([instance for instance in instances if instance.label_class == target_class]) if range_metric == "euclidean": - centers = np.array([dt.translation for dt in annos]) + centers = np.array([dt.translation for dt in instances]) filtered_annos = np.array([]) if centers.shape[0] > 0: dt_dists = np.linalg.norm(centers, axis=1) - filtered_annos = annos[dt_dists < max_dist] + filtered_annos = instances[dt_dists < max_dist] return filtered_annos -def get_ranks(dt_annos: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - """Get the rankings for the detection annotations. +def get_ranks(dts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get the rankings for the detections. Args: - dt_annos: Detection annotations. + dts: Detections. Returns: scores: The detection scores. ranks: The ranking for the detections. """ - scores = np.array([dt_anno.score for dt_anno in dt_annos]) + scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] return np.expand_dims(scores, 1)[ranks], ranks -def interp(prec: np.ndarray, method="all") -> np.ndarray: +def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: """Interpolate the precision over all recall levels. Args: @@ -72,19 +83,29 @@ def interp(prec: np.ndarray, method="all") -> np.ndarray: Returns: Interpolated precision at all recall levels. """ - if method == "all": - return np.maximum.accumulate(prec[::-1])[::-1] + if method == InterpType.ALL: + prec_interp = np.maximum.accumulate(prec[::-1])[::-1] + else: + raise NotImplemented("This interpolation method is not implemented!") + return prec_interp + +def compute_match_matrix(dts: np.ndarray, gts: np.ndarray, metric: SimFnType) -> np.ndarray: + """Calculate the match matrix between detections and ground truth labels, + using a specified similarity function. -def sim_fn(dt_annos: np.ndarray, gt_annos: np.ndarray, metric: SimFnType) -> np.ndarray: + Args: + dts: Detections. + gts: Ground truth labels. + metric: Similarity metric type. + + Returns: + Interpolated precision at all recall levels. + """ if metric == SimFnType.CENTER: - dt_centers = np.array([dt.translation for dt in dt_annos]) - gt_centers = np.array([gt.translation for gt in gt_annos]) + dt_centers = np.array([dt.translation for dt in dts]) + gt_centers = np.array([gt.translation for gt in gts]) sims = -cdist(dt_centers, gt_centers) - elif metric == SimFnType.IOU_2D: - raise NotImplemented("This similarity metric is not implemented!") - elif metric == SimFnType.IOU_3D: - raise NotImplemented("This similarity metric is not implemented!") else: raise NotImplemented("This similarity metric is not implemented!") return sims @@ -98,22 +119,22 @@ def get_error_types(match_scores: np.ndarray, thresh: float, metric: SimFnType) raise NotImplemented("This similarity metric is not implemented!") -def dist_fn(dt_df: pd.DataFrame, gt_df: pd.DataFrame, metric: DistFnType) -> np.ndarray: +def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndarray: if metric == DistFnType.TRANSLATION: - dt_centers = np.vstack(dt_df["translation"].array) - gt_centers = np.vstack(gt_df["translation"].array) + dt_centers = np.vstack(dts["translation"].array) + gt_centers = np.vstack(gts["translation"].array) trans_errors = np.linalg.norm(dt_centers - gt_centers, axis=1) return trans_errors elif metric == DistFnType.SCALE: - dt_dims = dt_df[["width", "length", "height"]] - gt_dims = gt_df[["width", "length", "height"]] + dt_dims = dts[["width", "length", "height"]] + gt_dims = gts[["width", "length", "height"]] inter = np.minimum(dt_dims, gt_dims).prod(axis=1) union = np.maximum(dt_dims, gt_dims).prod(axis=1) scale_errors = 1 - (inter / union) return scale_errors elif metric == DistFnType.ORIENTATION: - dt_yaws = R.from_quat(np.vstack(dt_df["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] - gt_yaws = R.from_quat(np.vstack(gt_df["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] + dt_yaws = R.from_quat(np.vstack(dts["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] + gt_yaws = R.from_quat(np.vstack(gts["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] orientation_errors = np.abs((dt_yaws - gt_yaws + np.pi) % (2 * np.pi) - np.pi) return orientation_errors else: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index e75e90e5..98de192a 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -2,26 +2,26 @@ import os from collections import defaultdict from dataclasses import dataclass +from multiprocessing import Pool from pathlib import Path from typing import List, Tuple +import click import matplotlib.pyplot as plt import numpy as np import pandas as pd -import click from argoverse.data_loading.object_label_record import read_label from argoverse.evaluation.detection_utils import ( DistFnType, SimFnType, + compute_match_matrix, dist_fn, - filter_annos, + filter_instances, get_error_types, get_ranks, interp, - sim_fn, ) -from tqdm.contrib.concurrent import process_map @dataclass @@ -32,7 +32,7 @@ class DetectionCfg: sim_ths: The similarity thresholds for determining a true positive. sim_fn_type: The type of similarity function to be used for calculating average precision. n_rec_samples: The number of recall points to sample uniformly in [0, 1]. - wdcs: The weight vector for the weighted detection composite score (WDCS). + dcs: The weight vector for the detection composite score (DCS). tp_threshold: The center distance threshold for the true positive metrics. significant_digits: The precision for metrics. """ @@ -40,7 +40,7 @@ class DetectionCfg: sim_ths: Tuple[float] = (0.5, 1.0, 2.0, 4.0) sim_fn_type: SimFnType = SimFnType.CENTER n_rec_samples: int = 101 - ads_weighting: Tuple[float] = (3, 1, 1, 1) + dcs_weighting: Tuple[float] = (3, 1, 1, 1) tp_thresh: float = 2.0 significant_digits: int = 3 @@ -75,8 +75,8 @@ def evaluate(self) -> pd.DataFrame: data = defaultdict(list) cls_inst_map = defaultdict(int) - chunk_size = max(1, len(gt_fpaths) // os.cpu_count()) - accum = process_map(self.accumulate, gt_fpaths, chunksize=chunk_size) + with Pool(os.cpu_count()) as p: + accum = p.map(self.accumulate, gt_fpaths) for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): data[cls_name].append(cls_stats) @@ -111,15 +111,15 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: ts = gt_fpath.stem.split("_")[-1] dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/tracked_object_labels_{ts}.json" - dt_annos = np.array(read_label(dt_fpath)) - gt_annos = np.array(read_label(gt_fpath)) - gt_clss = np.array([gt_anno.label_class for gt_anno in gt_annos]) + dts = np.array(read_label(dt_fpath)) + gts = np.array(read_label(gt_fpath)) + gt_clss = np.array([gt.label_class for gt in gts]) class_data = defaultdict(list) class_to_ninst = defaultdict(int) for gt_cls in np.unique(gt_clss): - dt_filtered = filter_annos(dt_annos, gt_cls) - gt_filtered = filter_annos(gt_annos, gt_cls) + dt_filtered = filter_instances(dts, gt_cls) + gt_filtered = filter_instances(gts, gt_cls) if dt_filtered.shape[0] > 0: error_types = self.assign(dt_filtered, gt_filtered) @@ -128,26 +128,26 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: class_to_ninst[gt_cls] = gt_filtered.shape[0] return class_data, class_to_ninst - def assign(self, dt_annos: np.ndarray, gt_annos: np.ndarray) -> List: - """Attempt assignment of each detection to a ground truth annotation. + def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: + """Attempt assignment of each detection to a ground truth label. Args: - dt_annos: Detection annotations. - gt_annos: Ground truth annotations. + dts: Detections. + gts: Ground truth labels. Returns: True positives, false positives, scores, and translation errors. """ n_threshs = len(self.dt_cfg.sim_ths) - error_types = np.zeros((dt_annos.shape[0], n_threshs + 3)) - scores, ranks = get_ranks(dt_annos) - if gt_annos.shape[0] == 0: + error_types = np.zeros((dts.shape[0], n_threshs + 3)) + scores, ranks = get_ranks(dts) + if gts.shape[0] == 0: return np.hstack((error_types, scores)) - match_matrix = sim_fn(dt_annos, gt_annos, self.dt_cfg.sim_fn_type) + match_matrix = compute_match_matrix(dts, gts, self.dt_cfg.sim_fn_type) - # Get the most similar GT annotation to each detection. + # Get the most similar GT label for each detection. gt_matches = np.expand_dims(match_matrix[ranks].argmax(axis=1), axis=0) # Grab the corresponding similarity score for each assignment. @@ -163,8 +163,8 @@ def assign(self, dt_annos: np.ndarray, gt_annos: np.ndarray) -> List: dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] - dt_df = pd.DataFrame([dt.__dict__ for dt in dt_annos[ranks][dt_tp_indices]]) - gt_df = pd.DataFrame([gt.__dict__ for gt in gt_annos[gt_tp_indices]]) + dt_df = pd.DataFrame([dt.__dict__ for dt in dts[ranks][dt_tp_indices]]) + gt_df = pd.DataFrame([gt.__dict__ for gt in gts[gt_tp_indices]]) trans_error = dist_fn(dt_df, gt_df, DistFnType.TRANSLATION) scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) @@ -217,10 +217,10 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict # TP Error Metrics tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + 3], axis=0) - ads_summands = np.hstack((ap, 1 - tp_metrics)) + dcs_summands = np.hstack((ap, 1 - tp_metrics)) # Ranking metric - wdcs = np.average(ads_summands, weights=self.dt_cfg.ads_weighting) + wdcs = np.average(dcs_summands, weights=self.dt_cfg.dcs_weighting) summary[cls_name] = [ap, *tp_metrics, wdcs] self.plot(rec_interp, prec_interp, cls_name) diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 546010db..064041c9 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -10,7 +10,7 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.evaluation.detection_utils import DistFnType, SimFnType, dist_fn, sim_fn +from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_match_matrix, dist_fn from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection" @@ -33,7 +33,7 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: def test_center_similarity() -> None: olr1 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) olr2 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) - assert sim_fn(olr1, olr2, SimFnType.CENTER) == -5 + assert compute_match_matrix(olr1, olr2, SimFnType.CENTER) == -5 def test_iou_2d_similarity() -> None: From 2de0952e7d421c766037266227a8043acac87229 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 9 Sep 2020 08:33:13 -0400 Subject: [PATCH 003/113] Cleaned up unused methods. --- argoverse/evaluation/detection_utils.py | 36 ++++++++++-------------- argoverse/evaluation/eval_detection.py | 4 +-- tests/test_data/json_save_test_file.json | 1 - tests/test_eval_detection.py | 10 ------- 4 files changed, 17 insertions(+), 34 deletions(-) delete mode 100644 tests/test_data/json_save_test_file.json diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 55b3caab..70ba264b 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -2,6 +2,11 @@ """Detection utilities for the Argoverse detection leaderboard. Accepts detections (in Argoverse ground truth format) and ground truth labels +for computing evaluation metrics for 3d object detection. We have five different, +metrics: mAP, ATE, ASE, AOE, and DCS. A true positive for mAP is defined as the +highest confidence prediction within a specified euclidean distance threshold +from a bird's-eye view. We prefer these metrics instead of IoU due to the +increased interpretability of the error modes in a set of detections. """ @@ -13,9 +18,6 @@ from scipy.spatial.distance import cdist from scipy.spatial.transform import Rotation as R -from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.utils.transform import quat2rotmat - class SimFnType(Enum): CENTER = auto() @@ -31,22 +33,29 @@ class InterpType(Enum): ALL = auto() +class FilterMetric(Enum): + EUCLIDEAN = auto() + + def filter_instances( - instances: np.ndarray, target_class: str, range_metric: str = "euclidean", max_dist: float = 50.0 + instances: np.ndarray, + target_class: str, + filter_metric: FilterMetric = FilterMetric.EUCLIDEAN, + max_dist: float = 50.0, ) -> np.ndarray: """Filter the annotations based on a set of conditions. Args: annos: The instances to be filtered. target_class: The name of the class of interest. - range_metric: The range metric used for filtering. + filter_metric: The range metric used for filtering. Returns: The filtered annotations. """ instances = np.array([instance for instance in instances if instance.label_class == target_class]) - if range_metric == "euclidean": + if filter_metric == FilterMetric.EUCLIDEAN: centers = np.array([dt.translation for dt in instances]) filtered_annos = np.array([]) @@ -139,18 +148,3 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar return orientation_errors else: raise NotImplemented("This distance metric is not implemented!") - - -def get_label_orientations(labels: List[ObjectLabelRecord]) -> float: - """Get the orientation (yaw) of a label. - - Args: - label: The label - - Returns: - The float orientation (yaw angle) of the label - """ - R = [quat2rotmat(l.quaternion) for l in labels] - v = np.array([1, 0, 0])[:, np.newaxis] - orientations = np.matmul(R, v) - return np.arctan2(orientations[:, 1, 0], orientations[:, 0, 0]) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 98de192a..6b9b8221 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -220,9 +220,9 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict dcs_summands = np.hstack((ap, 1 - tp_metrics)) # Ranking metric - wdcs = np.average(dcs_summands, weights=self.dt_cfg.dcs_weighting) + dcs = np.average(dcs_summands, weights=self.dt_cfg.dcs_weighting) - summary[cls_name] = [ap, *tp_metrics, wdcs] + summary[cls_name] = [ap, *tp_metrics, dcs] self.plot(rec_interp, prec_interp, cls_name) return summary diff --git a/tests/test_data/json_save_test_file.json b/tests/test_data/json_save_test_file.json deleted file mode 100644 index 0315c574..00000000 --- a/tests/test_data/json_save_test_file.json +++ /dev/null @@ -1 +0,0 @@ -{"a": 1, "b": null, "c": 9999, "d": "abc"} diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 064041c9..22c01f49 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -36,16 +36,6 @@ def test_center_similarity() -> None: assert compute_match_matrix(olr1, olr2, SimFnType.CENTER) == -5 -def test_iou_2d_similarity() -> None: - # TO DO - pass - - -def test_iou_3d_similarity() -> None: - # TO DO - pass - - def test_translation_distance() -> None: df1 = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) df2 = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) From 690ff3d6169cc8d96dfb2798ada25f34836a02cf Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 9 Sep 2020 15:08:47 -0400 Subject: [PATCH 004/113] Fix clipping. --- argoverse/evaluation/eval_detection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 6b9b8221..caa44288 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -40,7 +40,7 @@ class DetectionCfg: sim_ths: Tuple[float] = (0.5, 1.0, 2.0, 4.0) sim_fn_type: SimFnType = SimFnType.CENTER n_rec_samples: int = 101 - dcs_weighting: Tuple[float] = (3, 1, 1, 1) + cds_weights: Tuple[float] = (3, 1, 1, 1) tp_thresh: float = 2.0 significant_digits: int = 3 @@ -217,12 +217,12 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict # TP Error Metrics tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + 3], axis=0) - dcs_summands = np.hstack((ap, 1 - tp_metrics)) + cds_summands = np.hstack((ap, np.clip(1 - tp_metrics, a_min=0, a_max=None))) # Ranking metric - dcs = np.average(dcs_summands, weights=self.dt_cfg.dcs_weighting) + cds = np.average(cds_summands, weights=self.dt_cfg.cds_weights) - summary[cls_name] = [ap, *tp_metrics, dcs] + summary[cls_name] = [ap, *tp_metrics, cds] self.plot(rec_interp, prec_interp, cls_name) return summary From 07ab68cd6440cf37953a338690a163f7fcbc3691 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 10 Sep 2020 14:48:40 -0400 Subject: [PATCH 005/113] add some logging --- argoverse/evaluation/eval_detection.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index caa44288..79015327 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -2,6 +2,7 @@ import os from collections import defaultdict from dataclasses import dataclass +import logging from multiprocessing import Pool from pathlib import Path from typing import List, Tuple @@ -23,6 +24,8 @@ interp, ) +logger = logging.getLogger(__name__) + @dataclass class DetectionCfg: @@ -108,6 +111,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: """ log_id = gt_fpath.parents[1].stem + logger.info("log_id = %s", log_id) ts = gt_fpath.stem.split("_")[-1] dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/tracked_object_labels_{ts}.json" @@ -121,6 +125,8 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: dt_filtered = filter_instances(dts, gt_cls) gt_filtered = filter_instances(gts, gt_cls) + logger.info("%d detections" % dt_filtered.shape[0]) + logger.info("%d ground truth" % gt_filtered.shape[0]) if dt_filtered.shape[0] > 0: error_types = self.assign(dt_filtered, gt_filtered) class_data[gt_cls] = error_types @@ -224,6 +230,7 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict summary[cls_name] = [ap, *tp_metrics, cds] self.plot(rec_interp, prec_interp, cls_name) + logger.info("summary = %s" % summary) return summary def plot(self, rec_interp, prec_interp, cls_name) -> None: From ee9b37fadfc6701b71380cbff7f5620594f9c3af Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 10 Sep 2020 14:57:29 -0400 Subject: [PATCH 006/113] add test docstrings --- tests/test_eval_detection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 22c01f49..cd93b2b3 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -19,6 +19,7 @@ @pytest.fixture def evaluator() -> DetectionEvaluator: + """Define an evaluator that compares a set of results to itself.""" detection_cfg = DetectionCfg(significant_digits=32) return DetectionEvaluator( TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg @@ -66,18 +67,22 @@ def test_orientation_distance() -> None: def test_ap(metrics: DataFrame) -> None: + """Test that AP is 1 for the self-compared results.""" assert metrics.AP.Means == 1 def test_translation_error(metrics: DataFrame) -> None: + """Test that ATE is 0 for the self-compared results.""" assert metrics.ATE.Means == 0 def test_scale_error(metrics: DataFrame) -> None: + """Test that ASE is 0 for the self-compared results.""" assert metrics.ASE.Means == 0 def test_orientation_error(metrics: DataFrame) -> None: + """Test that AOE is 0 for the self-compared results.""" assert metrics.AOE.Means == 0 From 8cbb8988925568a441f085903ed0882d5b88c01c Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 10 Sep 2020 14:59:46 -0400 Subject: [PATCH 007/113] doc fixes --- argoverse/evaluation/eval_detection.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 79015327..56931b18 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -107,8 +107,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: gt_fpath: The ground truth file path. Returns: - The aggregated data used for summarization. - + The aggregated data used for summarization. """ log_id = gt_fpath.parents[1].stem logger.info("log_id = %s", log_id) @@ -143,7 +142,6 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: Returns: True positives, false positives, scores, and translation errors. - """ n_threshs = len(self.dt_cfg.sim_ths) error_types = np.zeros((dts.shape[0], n_threshs + 3)) @@ -191,7 +189,6 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict Returns: summary: The summary statistics. - """ summary = defaultdict(list) rec_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) @@ -234,6 +231,13 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict return summary def plot(self, rec_interp, prec_interp, cls_name) -> None: + """Plot and save the precision recall curve. + + Args: + rec_interp: The interpolated recall data. + prec_interp: The interpolated precision data. + cls_name: Class name. + """ plt.plot(rec_interp, prec_interp) plt.title("PR Curve") plt.xlabel("Recall") From 8b97bcc5ef50505161a1cca426a220ee2cab3a85 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 12 Sep 2020 21:30:07 -0400 Subject: [PATCH 008/113] fix pr comments --- argoverse/evaluation/detection_utils.py | 17 +++++++---------- argoverse/evaluation/eval_detection.py | 24 ++++++++++++++---------- argoverse/utils/transform.py | 14 +++++++++++--- tests/test_eval_detection.py | 17 +++++++++-------- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 70ba264b..1e84b8fe 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -18,6 +18,8 @@ from scipy.spatial.distance import cdist from scipy.spatial.transform import Rotation as R +from argoverse.utils.transform import quat_first2last + class SimFnType(Enum): CENTER = auto() @@ -120,14 +122,6 @@ def compute_match_matrix(dts: np.ndarray, gts: np.ndarray, metric: SimFnType) -> return sims -def get_error_types(match_scores: np.ndarray, thresh: float, metric: SimFnType) -> np.ndarray: - # Euclidean distance represented as a "similarity" metric. - if metric == SimFnType.CENTER: - return match_scores > -thresh - else: - raise NotImplemented("This similarity metric is not implemented!") - - def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndarray: if metric == DistFnType.TRANSLATION: dt_centers = np.vstack(dts["translation"].array) @@ -142,8 +136,11 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar scale_errors = 1 - (inter / union) return scale_errors elif metric == DistFnType.ORIENTATION: - dt_yaws = R.from_quat(np.vstack(dts["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] - gt_yaws = R.from_quat(np.vstack(gts["quaternion"].array)[:, [3, 0, 1, 2]]).as_euler("xyz")[:, 2] + # re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw + dt_yaws = R.from_quat(quat_first2last(np.vstack(dts["quaternion"].array))).as_euler("xyz")[:, 2] + gt_yaws = R.from_quat(quat_first2last(np.vstack(gts["quaternion"].array))).as_euler("xyz")[:, 2] + # the orientation distance is the absolute distance between the two yaws + # the '(d + pi) % 2pi - pi' is necessary to keep the distance within the interval [0, 2pi) orientation_errors = np.abs((dt_yaws - gt_yaws + np.pi) % (2 * np.pi) - np.pi) return orientation_errors else: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 56931b18..723c02ff 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -1,8 +1,8 @@ # +import logging import os from collections import defaultdict from dataclasses import dataclass -import logging from multiprocessing import Pool from pathlib import Path from typing import List, Tuple @@ -19,7 +19,6 @@ compute_match_matrix, dist_fn, filter_instances, - get_error_types, get_ranks, interp, ) @@ -33,10 +32,11 @@ class DetectionCfg: Args: sim_ths: The similarity thresholds for determining a true positive. - sim_fn_type: The type of similarity function to be used for calculating average precision. - n_rec_samples: The number of recall points to sample uniformly in [0, 1]. + sim_fn_type: The type of similarity function to be used for calculating + average precision. + n_rec_samples: Number of recall points to sample uniformly in [0, 1]. dcs: The weight vector for the detection composite score (DCS). - tp_threshold: The center distance threshold for the true positive metrics. + tp_threshold: Center distance threshold for the true positive metrics. significant_digits: The precision for metrics. """ @@ -112,7 +112,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: log_id = gt_fpath.parents[1].stem logger.info("log_id = %s", log_id) ts = gt_fpath.stem.split("_")[-1] - dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/tracked_object_labels_{ts}.json" + dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" dts = np.array(read_label(dt_fpath)) gts = np.array(read_label(gt_fpath)) @@ -157,13 +157,15 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: # Grab the corresponding similarity score for each assignment. match_scores = np.take_along_axis(match_matrix[ranks].T, gt_matches, axis=0).squeeze(0) - # Find the indices of the "first" detection assigned to each GT annotation. + # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) - for i, thresh in enumerate(self.dt_cfg.sim_ths): - tp_mask = get_error_types(match_scores[unique_dt_matches], thresh, self.dt_cfg.sim_fn_type) + for i, thr in enumerate(self.dt_cfg.sim_ths): + + # tp_mask may need to be defined differently with other similarity metrics + tp_mask = match_scores[unique_dt_matches] > -thr error_types[unique_dt_matches, i] = tp_mask - if thresh == self.dt_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: + if thr == self.dt_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] @@ -219,7 +221,9 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict # TP Error Metrics tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + 3], axis=0) + tp_metrics[2] /= np.pi / 2 # normalize orientation + # clip so that we don't get negative values in (1 - ATE) cds_summands = np.hstack((ap, np.clip(1 - tp_metrics, a_min=0, a_max=None))) # Ranking metric diff --git a/argoverse/utils/transform.py b/argoverse/utils/transform.py index 95448d8a..b262a439 100644 --- a/argoverse/utils/transform.py +++ b/argoverse/utils/transform.py @@ -26,6 +26,14 @@ def quat2rotmat(q: np.ndarray) -> np.ndarray: R: Array of shape (3, 3) representing a rotation matrix. """ assert np.isclose(np.linalg.norm(q), 1.0, atol=1e-12) - w, x, y, z = q - q_scipy = np.array([x, y, z, w]) - return Rotation.from_quat(q_scipy).as_dcm() + return Rotation.from_quat(quat_first2last(q)).as_dcm() + + +def quat_first2last(q: np.ndarray) -> np.ndarray: + """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in scipy.""" + return q[..., [1, 2, 3, 0]] + + +def quat_last2first(q: np.ndarray) -> np.ndarray: + """Re-order a scalar-last [x, y, z, w] quaternion format used by scipy to the scalar-first format in Argoverse.""" + return q[..., [3, 0, 1, 2]] diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index cd93b2b3..bca445ef 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -12,6 +12,7 @@ from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_match_matrix, dist_fn from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator +from argoverse.utils.transform import quat2rotmat, quat_last2first TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection" logging.getLogger("matplotlib.font_manager").disabled = True @@ -32,36 +33,40 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: def test_center_similarity() -> None: + """Test that the Center similarity function works.""" olr1 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) olr2 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) assert compute_match_matrix(olr1, olr2, SimFnType.CENTER) == -5 def test_translation_distance() -> None: + """Test that the translation distance function works.""" df1 = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) df2 = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) assert dist_fn(df1, df2, DistFnType.TRANSLATION) == 75 ** (1 / 2) def test_scale_distance() -> None: + """Test that the scale distance function works.""" df1 = DataFrame([{"width": 5, "height": 5, "length": 5}]) df2 = DataFrame([{"width": 10, "height": 10, "length": 10}]) assert (dist_fn(df1, df2, DistFnType.SCALE) == 1 - 0.125).all() def test_orientation_distance() -> None: + """Test that the orientation distance function works.""" # check all of the 45 degree angles vecs_45_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 4)] for i in range(len(vecs_45_apart) - 1): - df1 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_45_apart[i])}]) - df2 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_45_apart[i + 1])}]) + df1 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) + df2 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 4) assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 4) # check all of the 90 degree angles vecs_90_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 2)] for i in range(len(vecs_90_apart) - 1): - df1 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_90_apart[i])}]) - df2 = DataFrame([{"quaternion": _rotvec_to_quat(vecs_90_apart[i + 1])}]) + df1 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) + df2 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 2) assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 2) @@ -84,7 +89,3 @@ def test_scale_error(metrics: DataFrame) -> None: def test_orientation_error(metrics: DataFrame) -> None: """Test that AOE is 0 for the self-compared results.""" assert metrics.AOE.Means == 0 - - -def _rotvec_to_quat(rotvec: R) -> R: - return R.from_rotvec(rotvec).as_quat()[[3, 0, 1, 2]] From c85b5a7946a5e5bb2b4a75149273301b97f97ea6 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 12 Sep 2020 21:37:07 -0400 Subject: [PATCH 009/113] replace click with argparse --- argoverse/evaluation/eval_detection.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 723c02ff..9be97bc8 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -1,4 +1,5 @@ # +import argparse import logging import os from collections import defaultdict @@ -7,7 +8,6 @@ from pathlib import Path from typing import List, Tuple -import click import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -250,11 +250,14 @@ def plot(self, rec_interp, prec_interp, cls_name) -> None: plt.close() -@click.command() -@click.option("--dt_fpath", "-d", help="Detection root folder path.", type=click.Path(exists=True), required=True) -@click.option("--gt_fpath", "-g", help="Ground truth root folder path.", type=click.Path(exists=True), required=True) -@click.option("--fig_fpath", "-f", help="Figures root folder path.", type=str, default="figs/") def main(dt_fpath: str, gt_fpath: str, fig_fpath: str) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-d", "--dt_fpath", type=str, help="Detection root folder path.", required=True) + parser.add_argument("-g", "--gt_fpath", type=str, help="Ground truth root folder path.", required=True) + parser.add_argument("-f", "--fig_fpath", type=str, help="Figures root folder path.", default="figs/") + args = parser.parse_args() + logger.info("args == %s", args) + dt_fpath = Path(dt_fpath) gt_fpath = Path(gt_fpath) fig_fpath = Path(fig_fpath) From d28d01e4d6710f454aea1b1d64097d2df0dfba97 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 12 Sep 2020 21:49:54 -0400 Subject: [PATCH 010/113] add missing types to function header --- argoverse/evaluation/eval_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 9be97bc8..3cfdd5ca 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -42,7 +42,7 @@ class DetectionCfg: sim_ths: Tuple[float] = (0.5, 1.0, 2.0, 4.0) sim_fn_type: SimFnType = SimFnType.CENTER - n_rec_samples: int = 101 + n_rec_samples: int = 101 # 101 gives us one sample at ever integer from 0 to 100 cds_weights: Tuple[float] = (3, 1, 1, 1) tp_thresh: float = 2.0 significant_digits: int = 3 @@ -234,7 +234,7 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict logger.info("summary = %s" % summary) return summary - def plot(self, rec_interp, prec_interp, cls_name) -> None: + def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) -> None: """Plot and save the precision recall curve. Args: From 8f08b674e5c9f23608f3c526003256b7856685c8 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 12 Sep 2020 21:55:29 -0400 Subject: [PATCH 011/113] doc for dist_fn --- argoverse/evaluation/detection_utils.py | 10 ++++++++++ argoverse/evaluation/eval_detection.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 1e84b8fe..43da93ef 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -123,6 +123,16 @@ def compute_match_matrix(dts: np.ndarray, gts: np.ndarray, metric: SimFnType) -> def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndarray: + """Distance functions between detections and ground truth. + + Args: + dts: Detections. + gts: Ground truth labels. + metric: Distance function type. + + Returns: + Distance between the detections and ground truth, using the provided metric. + """ if metric == DistFnType.TRANSLATION: dt_centers = np.vstack(dts["translation"].array) gt_centers = np.vstack(gts["translation"].array) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 3cfdd5ca..f998308c 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -44,7 +44,7 @@ class DetectionCfg: sim_fn_type: SimFnType = SimFnType.CENTER n_rec_samples: int = 101 # 101 gives us one sample at ever integer from 0 to 100 cds_weights: Tuple[float] = (3, 1, 1, 1) - tp_thresh: float = 2.0 + tp_thresh: float = 2.0 # in meters significant_digits: int = 3 From 5f532fce77981b0a40d45543b39f234b252c93a9 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 14 Sep 2020 15:54:12 +0000 Subject: [PATCH 012/113] Fixed a few argument issues. --- argoverse/evaluation/detection_utils.py | 13 ++++++++----- argoverse/evaluation/eval_detection.py | 22 ++++++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 43da93ef..f1e1b0af 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -51,7 +51,7 @@ def filter_instances( annos: The instances to be filtered. target_class: The name of the class of interest. filter_metric: The range metric used for filtering. - + Returns: The filtered annotations. """ @@ -90,7 +90,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: Args: prec: Precision at all recall levels. method: Accumulation method. - + Returns: Interpolated precision at all recall levels. """ @@ -109,7 +109,7 @@ def compute_match_matrix(dts: np.ndarray, gts: np.ndarray, metric: SimFnType) -> dts: Detections. gts: Ground truth labels. metric: Similarity metric type. - + Returns: Interpolated precision at all recall levels. """ @@ -147,8 +147,11 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar return scale_errors elif metric == DistFnType.ORIENTATION: # re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw - dt_yaws = R.from_quat(quat_first2last(np.vstack(dts["quaternion"].array))).as_euler("xyz")[:, 2] - gt_yaws = R.from_quat(quat_first2last(np.vstack(gts["quaternion"].array))).as_euler("xyz")[:, 2] + dt_quats = np.vstack(dts["quaternion"].array) + dt_yaws = R.from_quat(quat_first2last(dt_quats)).as_euler("xyz")[:, 2] + + gt_quats = np.vstack(gts["quaternion"].array) + gt_yaws = R.from_quat(quat_first2last(gt_quats)).as_euler("xyz")[:, 2] # the orientation distance is the absolute distance between the two yaws # the '(d + pi) % 2pi - pi' is necessary to keep the distance within the interval [0, 2pi) orientation_errors = np.abs((dt_yaws - gt_yaws + np.pi) % (2 * np.pi) - np.pi) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index f998308c..011dce8d 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import List, Tuple +import matplotlib import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -23,6 +24,7 @@ interp, ) +matplotlib.use("Agg") logger = logging.getLogger(__name__) @@ -61,7 +63,7 @@ class DetectionEvaluator: dt_fpath: Path = Path("detections") gt_fpath: Path = Path("/data/argoverse") - figures_fpath: Path = Path("figs") + fig_fpath: Path = Path("figs") dt_cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: @@ -107,7 +109,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: gt_fpath: The ground truth file path. Returns: - The aggregated data used for summarization. + The aggregated data used for summarization. """ log_id = gt_fpath.parents[1].stem logger.info("log_id = %s", log_id) @@ -195,8 +197,8 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict summary = defaultdict(list) rec_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) num_ths = len(self.dt_cfg.sim_ths) - if not self.figures_fpath.is_dir(): - self.figures_fpath.mkdir(parents=True, exist_ok=True) + if not self.fig_fpath.is_dir(): + self.fig_fpath.mkdir(parents=True, exist_ok=True) for cls_name, cls_stats in data.items(): num_inst = cls_inst_map[cls_name] ranks = cls_stats[:, -1].argsort()[::-1] @@ -246,11 +248,11 @@ def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) - plt.title("PR Curve") plt.xlabel("Recall") plt.ylabel("Precision") - plt.savefig(f"{self.figures_fpath}/{cls_name}.png") + plt.savefig(f"{self.fig_fpath}/{cls_name}.png") plt.close() -def main(dt_fpath: str, gt_fpath: str, fig_fpath: str) -> None: +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("-d", "--dt_fpath", type=str, help="Detection root folder path.", required=True) parser.add_argument("-g", "--gt_fpath", type=str, help="Ground truth root folder path.", required=True) @@ -258,11 +260,11 @@ def main(dt_fpath: str, gt_fpath: str, fig_fpath: str) -> None: args = parser.parse_args() logger.info("args == %s", args) - dt_fpath = Path(dt_fpath) - gt_fpath = Path(gt_fpath) - fig_fpath = Path(fig_fpath) + dt_fpath = Path(args.dt_fpath) + gt_fpath = Path(args.gt_fpath) + fig_fpath = Path(args.fig_fpath) - evaluator = DetectionEvaluator(dt_fpath, gt_fpath) + evaluator = DetectionEvaluator(dt_fpath, gt_fpath, fig_fpath) metrics = evaluator.evaluate() print(metrics) From 9707b4e50ee237a9eb7fd02ca784f21c2841fa3a Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 14 Sep 2020 22:01:20 +0000 Subject: [PATCH 013/113] Added angle normalization method. --- argoverse/evaluation/detection_utils.py | 20 +++++++++++++++++--- argoverse/evaluation/eval_detection.py | 3 ++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index f1e1b0af..f12fb3d6 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -152,9 +152,23 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar gt_quats = np.vstack(gts["quaternion"].array) gt_yaws = R.from_quat(quat_first2last(gt_quats)).as_euler("xyz")[:, 2] - # the orientation distance is the absolute distance between the two yaws - # the '(d + pi) % 2pi - pi' is necessary to keep the distance within the interval [0, 2pi) - orientation_errors = np.abs((dt_yaws - gt_yaws + np.pi) % (2 * np.pi) - np.pi) + + signed_orientation_errors = normalize_angle(dt_yaws - gt_yaws) + orientation_errors = np.abs(signed_orientation_errors) return orientation_errors else: raise NotImplemented("This distance metric is not implemented!") + + +def normalize_angle(angle: float) -> float: + """Map angle (in radians) to [0, 2π]. + + Args: + angle: Angle (in radians). + + Returns: + The angle (in radians) mapped to the interval [0, 2π]. + """ + period = 2 * np.pi + phase_shift = np.pi + return (angle + np.pi) % period - phase_shift diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 011dce8d..b8b6a192 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -215,7 +215,8 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict precs = interp(precs) prec_interp = np.interp(rec_interp, recs, precs, right=0) - ap_th = prec_interp.mean() + prec_trun = prec_interp[10 + 1 :] + ap_th = prec_trun.mean() summary[cls_name] += [ap_th] # AP Metric From 83b4ef2cddf0c1791bf1c20a694d70c140a9beda Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 14 Sep 2020 23:36:57 +0000 Subject: [PATCH 014/113] Temporary fix (potentially). --- argoverse/evaluation/detection_utils.py | 6 +++--- argoverse/utils/transform.py | 17 ++++++++++++----- tests/test_eval_detection.py | 10 +++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index f12fb3d6..95d48c87 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -18,7 +18,7 @@ from scipy.spatial.distance import cdist from scipy.spatial.transform import Rotation as R -from argoverse.utils.transform import quat_first2last +from argoverse.utils.transform import quat_argo2scipy_vectorized class SimFnType(Enum): @@ -148,10 +148,10 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar elif metric == DistFnType.ORIENTATION: # re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw dt_quats = np.vstack(dts["quaternion"].array) - dt_yaws = R.from_quat(quat_first2last(dt_quats)).as_euler("xyz")[:, 2] + dt_yaws = R.from_quat(quat_argo2scipy_vectorized(dt_quats)).as_euler("xyz")[:, 2] gt_quats = np.vstack(gts["quaternion"].array) - gt_yaws = R.from_quat(quat_first2last(gt_quats)).as_euler("xyz")[:, 2] + gt_yaws = R.from_quat(quat_argo2scipy_vectorized(gt_quats)).as_euler("xyz")[:, 2] signed_orientation_errors = normalize_angle(dt_yaws - gt_yaws) orientation_errors = np.abs(signed_orientation_errors) diff --git a/argoverse/utils/transform.py b/argoverse/utils/transform.py index b262a439..8d4c945d 100644 --- a/argoverse/utils/transform.py +++ b/argoverse/utils/transform.py @@ -26,14 +26,21 @@ def quat2rotmat(q: np.ndarray) -> np.ndarray: R: Array of shape (3, 3) representing a rotation matrix. """ assert np.isclose(np.linalg.norm(q), 1.0, atol=1e-12) - return Rotation.from_quat(quat_first2last(q)).as_dcm() + return Rotation.from_quat(quat_argo2scipy(q)).as_dcm() -def quat_first2last(q: np.ndarray) -> np.ndarray: - """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in scipy.""" +def quat_argo2scipy(q: np.ndarray) -> np.ndarray: + """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" + w, x, y, z = q + q_scipy = np.array([x, y, z, w]) + return q_scipy + + +def quat_argo2scipy_vectorized(q: np.ndarray) -> np.ndarray: + """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" return q[..., [1, 2, 3, 0]] -def quat_last2first(q: np.ndarray) -> np.ndarray: - """Re-order a scalar-last [x, y, z, w] quaternion format used by scipy to the scalar-first format in Argoverse.""" +def quat_scipy2argo_vectorized(q: np.ndarray) -> np.ndarray: + """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" return q[..., [3, 0, 1, 2]] diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index bca445ef..45b2e463 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -12,7 +12,7 @@ from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_match_matrix, dist_fn from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator -from argoverse.utils.transform import quat2rotmat, quat_last2first +from argoverse.utils.transform import quat_scipy2argo_vectorized TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection" logging.getLogger("matplotlib.font_manager").disabled = True @@ -58,15 +58,15 @@ def test_orientation_distance() -> None: # check all of the 45 degree angles vecs_45_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 4)] for i in range(len(vecs_45_apart) - 1): - df1 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) - df2 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) + df1 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) + df2 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 4) assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 4) # check all of the 90 degree angles vecs_90_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 2)] for i in range(len(vecs_90_apart) - 1): - df1 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) - df2 = DataFrame([{"quaternion": quat_last2first(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) + df1 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) + df2 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 2) assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 2) From fdf1e9086f534cc02336094f3f2a8551c0d775bf Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 16 Sep 2020 23:54:03 +0000 Subject: [PATCH 015/113] Typing fixes + other enhancements. --- argoverse/evaluation/detection_utils.py | 5 +- argoverse/evaluation/eval_detection.py | 114 +++++++++++++----------- tests/test_eval_detection.py | 10 +-- 3 files changed, 72 insertions(+), 57 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 95d48c87..e3d31097 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -45,12 +45,13 @@ def filter_instances( filter_metric: FilterMetric = FilterMetric.EUCLIDEAN, max_dist: float = 50.0, ) -> np.ndarray: - """Filter the annotations based on a set of conditions. + """Filter the GT annotations based on a set of conditions (e.g., distance from egovehicle). Args: - annos: The instances to be filtered. + instances: The instances to be filtered. target_class: The name of the class of interest. filter_metric: The range metric used for filtering. + max_dist: The maximum distance for range filtering. Returns: The filtered annotations. diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index b8b6a192..6073caef 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -3,16 +3,17 @@ import logging import os from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field from multiprocessing import Pool from pathlib import Path -from typing import List, Tuple +from typing import DefaultDict, List, Tuple import matplotlib import matplotlib.pyplot as plt import numpy as np import pandas as pd +from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT from argoverse.data_loading.object_label_record import read_label from argoverse.evaluation.detection_utils import ( DistFnType, @@ -36,18 +37,20 @@ class DetectionCfg: sim_ths: The similarity thresholds for determining a true positive. sim_fn_type: The type of similarity function to be used for calculating average precision. - n_rec_samples: Number of recall points to sample uniformly in [0, 1]. + n_rec_samples: Number of recall points to sample uniformly in [0, 1]. + Default to 100 recall samples. dcs: The weight vector for the detection composite score (DCS). - tp_threshold: Center distance threshold for the true positive metrics. + tp_threshold: Center distance threshold for the true positive metrics (in meters). significant_digits: The precision for metrics. """ - sim_ths: Tuple[float] = (0.5, 1.0, 2.0, 4.0) + sim_ths: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) sim_fn_type: SimFnType = SimFnType.CENTER - n_rec_samples: int = 101 # 101 gives us one sample at ever integer from 0 to 100 - cds_weights: Tuple[float] = (3, 1, 1, 1) - tp_thresh: float = 2.0 # in meters + n_rec_samples: int = 101 + cds_weights: List[float] = field(default_factory=lambda: [3, 1, 1, 1]) + tp_thresh: float = 2.0 significant_digits: int = 3 + detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) @dataclass @@ -67,73 +70,77 @@ class DetectionEvaluator: dt_cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: - """Evaluate detection output and return metrics. + """Evaluate detection output and return metrics. The multiprocessing + library is used for parallel assignment between detections and ground truth + annotations. Returns: The evaluation metrics. """ - dt_fpaths = list(self.dt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) gt_fpaths = list(self.gt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) assert len(dt_fpaths) == len(gt_fpaths) - data = defaultdict(list) - cls_inst_map = defaultdict(int) + data: DefaultDict[str, np.ndarray] = defaultdict(list) + cls_to_ninst: DefaultDict[str, int] = defaultdict(int) with Pool(os.cpu_count()) as p: accum = p.map(self.accumulate, gt_fpaths) + # [self.accumulate(gt_fpath) for gt_fpath in gt_fpaths] + for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): data[cls_name].append(cls_stats) for cls_name, num_inst in frame_cls_to_inst.items(): - cls_inst_map[cls_name] += num_inst + cls_to_ninst[cls_name] += num_inst data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) + init_data = {k: [0, 1.0, 1.0, 1.0, 0] for k in self.dt_cfg.detection_classes} + + columns = ["AP", "ATE", "ASE", "AOE", "CDS"] + summary = pd.DataFrame.from_dict(init_data, orient="index", columns=columns) - summary = self.summarize(data, cls_inst_map) - summary = ( - pd.DataFrame.from_dict(summary, orient="index", columns=["AP", "ATE", "ASE", "AOE", "WDCS"]) - .round(self.dt_cfg.significant_digits) - .sort_index() - ) - summary.index = summary.index.str.capitalize() + summary_update = pd.DataFrame.from_dict(self.summarize(data, cls_to_ninst), orient="index", columns=columns) - summary.loc[""] = ["", "", "", "", ""] - summary.loc["Means"] = summary.iloc[:-1].mean().round(self.dt_cfg.significant_digits) + summary.update(summary_update) + summary = summary.round(self.dt_cfg.significant_digits) + summary.index = summary.index.str.title() + + summary.loc["Average Metrics"] = summary.mean().round(self.dt_cfg.significant_digits) return summary def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: """Accumulate the statistics for each LiDAR frame. Args: - gt_fpath: The ground truth file path. + gt_fpath: Ground truth file path. Returns: - The aggregated data used for summarization. + cls_stats: Class statistics of shape (N, 8) + cls_to_ninst: Mapping of the class names to the number of instances in the ground + truth dataset. """ log_id = gt_fpath.parents[1].stem - logger.info("log_id = %s", log_id) + logger.info(f"log_id = {log_id}") ts = gt_fpath.stem.split("_")[-1] dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" dts = np.array(read_label(dt_fpath)) gts = np.array(read_label(gt_fpath)) - gt_clss = np.array([gt.label_class for gt in gts]) - class_data = defaultdict(list) - class_to_ninst = defaultdict(int) - for gt_cls in np.unique(gt_clss): - dt_filtered = filter_instances(dts, gt_cls) - gt_filtered = filter_instances(gts, gt_cls) + cls_stats = defaultdict(list) + cls_to_ninst = defaultdict(int) + for dt_cls in self.dt_cfg.detection_classes: + dt_filtered = filter_instances(dts, dt_cls) + gt_filtered = filter_instances(gts, dt_cls) logger.info("%d detections" % dt_filtered.shape[0]) logger.info("%d ground truth" % gt_filtered.shape[0]) if dt_filtered.shape[0] > 0: - error_types = self.assign(dt_filtered, gt_filtered) - class_data[gt_cls] = error_types + cls_stats[dt_cls] = self.assign(dt_filtered, gt_filtered) - class_to_ninst[gt_cls] = gt_filtered.shape[0] - return class_data, class_to_ninst + cls_to_ninst[dt_cls] = gt_filtered.shape[0] + return cls_stats, cls_to_ninst def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: """Attempt assignment of each detection to a ground truth label. @@ -184,41 +191,48 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: return np.hstack((error_types, scores)) - def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict: + def summarize( + self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] + ) -> DefaultDict[str, List]: """Calculate and print the detection metrics. Args: data: The aggregated data used for summarization. - cls_inst_map: Map of classes to number of instances. + cls_to_ninst: Map of classes to number of instances. Returns: summary: The summary statistics. """ - summary = defaultdict(list) - rec_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) + summary: DefaultDict[str, List] = defaultdict(list) + recalls_interpolated = np.linspace(0, 1, self.dt_cfg.n_rec_samples) num_ths = len(self.dt_cfg.sim_ths) if not self.fig_fpath.is_dir(): self.fig_fpath.mkdir(parents=True, exist_ok=True) + for cls_name, cls_stats in data.items(): - num_inst = cls_inst_map[cls_name] + ninst = cls_to_ninst[cls_name] ranks = cls_stats[:, -1].argsort()[::-1] cls_stats = cls_stats[ranks] + for i, _ in enumerate(self.dt_cfg.sim_ths): tp = cls_stats[:, i].astype(bool) cumulative_tp = np.cumsum(tp, dtype=np.int) cumulative_fp = np.cumsum(~tp, dtype=np.int) - cumulative_fn = num_inst - cumulative_tp + cumulative_fn = ninst - cumulative_tp + + precisions = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) + recalls = cumulative_tp / (cumulative_tp + cumulative_fn) - precs = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) - recs = cumulative_tp / (cumulative_tp + cumulative_fn) + precisions = interp(precisions) + precisions_interpolated = np.interp(recalls_interpolated, recalls, precisions, right=0) - precs = interp(precs) - prec_interp = np.interp(rec_interp, recs, precs, right=0) - prec_trun = prec_interp[10 + 1 :] - ap_th = prec_trun.mean() + prec_truncated = precisions_interpolated[10 + 1 :] + ap_th = prec_truncated.mean() summary[cls_name] += [ap_th] + self.plot(recalls_interpolated, precisions_interpolated, cls_name) + # AP Metric ap = np.array(summary[cls_name][:num_ths]).mean() @@ -233,8 +247,8 @@ def summarize(self, data: defaultdict, cls_inst_map: defaultdict) -> defaultdict cds = np.average(cds_summands, weights=self.dt_cfg.cds_weights) summary[cls_name] = [ap, *tp_metrics, cds] - self.plot(rec_interp, prec_interp, cls_name) - logger.info("summary = %s" % summary) + + logger.info(f"summary = {summary}") return summary def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) -> None: @@ -259,7 +273,7 @@ def main() -> None: parser.add_argument("-g", "--gt_fpath", type=str, help="Ground truth root folder path.", required=True) parser.add_argument("-f", "--fig_fpath", type=str, help="Figures root folder path.", default="figs/") args = parser.parse_args() - logger.info("args == %s", args) + logger.info(f"args == {args}") dt_fpath = Path(args.dt_fpath) gt_fpath = Path(args.gt_fpath) diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 45b2e463..fc84d6da 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -21,7 +21,7 @@ @pytest.fixture def evaluator() -> DetectionEvaluator: """Define an evaluator that compares a set of results to itself.""" - detection_cfg = DetectionCfg(significant_digits=32) + detection_cfg = DetectionCfg(significant_digits=32, detection_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) @@ -73,19 +73,19 @@ def test_orientation_distance() -> None: def test_ap(metrics: DataFrame) -> None: """Test that AP is 1 for the self-compared results.""" - assert metrics.AP.Means == 1 + assert metrics.AP.loc["Average Metrics"] == 1 def test_translation_error(metrics: DataFrame) -> None: """Test that ATE is 0 for the self-compared results.""" - assert metrics.ATE.Means == 0 + assert metrics.ATE.loc["Average Metrics"] == 0 def test_scale_error(metrics: DataFrame) -> None: """Test that ASE is 0 for the self-compared results.""" - assert metrics.ASE.Means == 0 + assert metrics.ASE.loc["Average Metrics"] == 0 def test_orientation_error(metrics: DataFrame) -> None: """Test that AOE is 0 for the self-compared results.""" - assert metrics.AOE.Means == 0 + assert metrics.AOE.loc["Average Metrics"] == 0 From 8509df43d772cb6d1504875d0d3ce65218a9d218 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 04:01:57 +0000 Subject: [PATCH 016/113] Type fixes + matplotlib fix. --- argoverse/evaluation/detection_utils.py | 6 ++-- argoverse/evaluation/eval_detection.py | 43 ++++++++++++++----------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index e3d31097..bf3528b2 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -4,7 +4,7 @@ Accepts detections (in Argoverse ground truth format) and ground truth labels for computing evaluation metrics for 3d object detection. We have five different, metrics: mAP, ATE, ASE, AOE, and DCS. A true positive for mAP is defined as the -highest confidence prediction within a specified euclidean distance threshold +highest confidence prediction within a specified Euclidean distance threshold from a bird's-eye view. We prefer these metrics instead of IoU due to the increased interpretability of the error modes in a set of detections. @@ -162,13 +162,13 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar def normalize_angle(angle: float) -> float: - """Map angle (in radians) to [0, 2π]. + """Map angle (in radians) to [0, 2π). Args: angle: Angle (in radians). Returns: - The angle (in radians) mapped to the interval [0, 2π]. + The angle (in radians) mapped to the interval [0, 2π). """ period = 2 * np.pi phase_shift = np.pi diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 6073caef..f1b67aa7 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -9,7 +9,6 @@ from typing import DefaultDict, List, Tuple import matplotlib -import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -25,7 +24,12 @@ interp, ) -matplotlib.use("Agg") +matplotlib.use("Agg") # isort:skip + + +import matplotlib.pyplot as plt # isort:skip + + logger = logging.getLogger(__name__) @@ -37,8 +41,7 @@ class DetectionCfg: sim_ths: The similarity thresholds for determining a true positive. sim_fn_type: The type of similarity function to be used for calculating average precision. - n_rec_samples: Number of recall points to sample uniformly in [0, 1]. - Default to 100 recall samples. + n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. dcs: The weight vector for the detection composite score (DCS). tp_threshold: Center distance threshold for the true positive metrics (in meters). significant_digits: The precision for metrics. @@ -51,6 +54,7 @@ class DetectionCfg: tp_thresh: float = 2.0 significant_digits: int = 3 detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) + save_figs: bool = False @dataclass @@ -86,7 +90,6 @@ def evaluate(self) -> pd.DataFrame: with Pool(os.cpu_count()) as p: accum = p.map(self.accumulate, gt_fpaths) - # [self.accumulate(gt_fpath) for gt_fpath in gt_fpaths] for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): @@ -95,11 +98,12 @@ def evaluate(self) -> pd.DataFrame: cls_to_ninst[cls_name] += num_inst data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) - init_data = {k: [0, 1.0, 1.0, 1.0, 0] for k in self.dt_cfg.detection_classes} columns = ["AP", "ATE", "ASE", "AOE", "CDS"] - summary = pd.DataFrame.from_dict(init_data, orient="index", columns=columns) + # Initialize metrics table [0, 1.0, 1.0, 1.0, 0] represents the default value for each column. + init_data = {k: [0, 1.0, 1.0, 1.0, 0] for k in self.dt_cfg.detection_classes} + summary = pd.DataFrame.from_dict(init_data, orient="index", columns=columns) summary_update = pd.DataFrame.from_dict(self.summarize(data, cls_to_ninst), orient="index", columns=columns) summary.update(summary_update) @@ -109,7 +113,7 @@ def evaluate(self) -> pd.DataFrame: summary.loc["Average Metrics"] = summary.mean().round(self.dt_cfg.significant_digits) return summary - def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: + def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: """Accumulate the statistics for each LiDAR frame. Args: @@ -134,20 +138,20 @@ def accumulate(self, gt_fpath: Path) -> Tuple[np.ndarray, np.ndarray]: dt_filtered = filter_instances(dts, dt_cls) gt_filtered = filter_instances(gts, dt_cls) - logger.info("%d detections" % dt_filtered.shape[0]) - logger.info("%d ground truth" % gt_filtered.shape[0]) + logger.info(f"{dt_filtered.shape[0]} detections") + logger.info(f"{gt_filtered.shape[0]} ground truth") if dt_filtered.shape[0] > 0: cls_stats[dt_cls] = self.assign(dt_filtered, gt_filtered) cls_to_ninst[dt_cls] = gt_filtered.shape[0] return cls_stats, cls_to_ninst - def assign(self, dts: np.ndarray, gts: np.ndarray) -> List: + def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: """Attempt assignment of each detection to a ground truth label. Args: - dts: Detections. - gts: Ground truth labels. + dts: Detections of shape (N,). + gts: Ground truth labels of shape (M,). Returns: True positives, false positives, scores, and translation errors. @@ -204,7 +208,7 @@ def summarize( summary: The summary statistics. """ summary: DefaultDict[str, List] = defaultdict(list) - recalls_interpolated = np.linspace(0, 1, self.dt_cfg.n_rec_samples) + recalls_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) num_ths = len(self.dt_cfg.sim_ths) if not self.fig_fpath.is_dir(): self.fig_fpath.mkdir(parents=True, exist_ok=True) @@ -225,13 +229,14 @@ def summarize( recalls = cumulative_tp / (cumulative_tp + cumulative_fn) precisions = interp(precisions) - precisions_interpolated = np.interp(recalls_interpolated, recalls, precisions, right=0) + precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) - prec_truncated = precisions_interpolated[10 + 1 :] + prec_truncated = precisions_interp[10 + 1 :] ap_th = prec_truncated.mean() summary[cls_name] += [ap_th] - self.plot(recalls_interpolated, precisions_interpolated, cls_name) + if self.dt_cfg.save_figs: + self.plot(recalls_interp, precisions_interp, cls_name) # AP Metric ap = np.array(summary[cls_name][:num_ths]).mean() @@ -255,8 +260,8 @@ def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) - """Plot and save the precision recall curve. Args: - rec_interp: The interpolated recall data. - prec_interp: The interpolated precision data. + rec_interp: Interpolated recall data of shape (101,). + prec_interp: Interpolated precision data of shape (101,). cls_name: Class name. """ plt.plot(rec_interp, prec_interp) From b7abaeaf731afc8b1da872782a1146b1dc1cf2a0 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 14:25:14 +0000 Subject: [PATCH 017/113] Added additional shape information. --- argoverse/evaluation/detection_utils.py | 17 +++++++++-------- argoverse/evaluation/eval_detection.py | 9 +++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index bf3528b2..9c1a61c2 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -18,6 +18,7 @@ from scipy.spatial.distance import cdist from scipy.spatial.transform import Rotation as R +from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.utils.transform import quat_argo2scipy_vectorized @@ -69,15 +70,15 @@ def filter_instances( return filtered_annos -def get_ranks(dts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: +def get_ranks(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: """Get the rankings for the detections. Args: - dts: Detections. + dts: Detections (N,). Returns: - scores: The detection scores. - ranks: The ranking for the detections. + scores: The detection scores (N,). + ranks: The ranking for the detections (N,). """ scores = np.array([dt.score for dt in dts]) @@ -89,7 +90,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: """Interpolate the precision over all recall levels. Args: - prec: Precision at all recall levels. + prec: Precision at all recall levels (N,). method: Accumulation method. Returns: @@ -102,13 +103,13 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: return prec_interp -def compute_match_matrix(dts: np.ndarray, gts: np.ndarray, metric: SimFnType) -> np.ndarray: +def compute_match_matrix(dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: SimFnType) -> np.ndarray: """Calculate the match matrix between detections and ground truth labels, using a specified similarity function. Args: - dts: Detections. - gts: Ground truth labels. + dts: Detections (N,). + gts: Ground truth labels (M,). metric: Similarity metric type. Returns: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index f1b67aa7..cf10c556 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -127,10 +127,11 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa log_id = gt_fpath.parents[1].stem logger.info(f"log_id = {log_id}") ts = gt_fpath.stem.split("_")[-1] + dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" - dts = np.array(read_label(dt_fpath)) - gts = np.array(read_label(gt_fpath)) + dts = np.array(read_label(str(dt_fpath))) + gts = np.array(read_label(str(gt_fpath))) cls_stats = defaultdict(list) cls_to_ninst = defaultdict(int) @@ -260,8 +261,8 @@ def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) - """Plot and save the precision recall curve. Args: - rec_interp: Interpolated recall data of shape (101,). - prec_interp: Interpolated precision data of shape (101,). + rec_interp: Interpolated recall data of shape (N,). + prec_interp: Interpolated precision data of shape (N,). cls_name: Class name. """ plt.plot(rec_interp, prec_interp) From e8d8daa58a696959ddb402095d4be6411da5669e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 14:48:55 +0000 Subject: [PATCH 018/113] Added additional shape info. --- argoverse/evaluation/detection_utils.py | 7 ++++--- argoverse/evaluation/eval_detection.py | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 9c1a61c2..b7ed23e1 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -19,6 +19,7 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord +from argoverse.evaluation.eval_tracking import get_orientation_error_deg from argoverse.utils.transform import quat_argo2scipy_vectorized @@ -113,7 +114,7 @@ def compute_match_matrix(dts: List[ObjectLabelRecord], gts: List[ObjectLabelReco metric: Similarity metric type. Returns: - Interpolated precision at all recall levels. + sims: Similarity scores between detections and ground truth annotations (N, M). """ if metric == SimFnType.CENTER: dt_centers = np.array([dt.translation for dt in dts]) @@ -163,10 +164,10 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar def normalize_angle(angle: float) -> float: - """Map angle (in radians) to [0, 2π). + """Map angle (in radians) from domain [-π, π] to [0, 2π). Args: - angle: Angle (in radians). + angle: Angle (in radians) in domain [-π, π]. Returns: The angle (in radians) mapped to the interval [0, 2π). diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index cf10c556..efe28dd4 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -3,7 +3,7 @@ import logging import os from collections import defaultdict -from dataclasses import dataclass, field +from dataclasses import MISSING, dataclass, field from multiprocessing import Pool from pathlib import Path from typing import DefaultDict, List, Tuple @@ -68,9 +68,9 @@ class DetectionEvaluator: dt_cfg: The detection configuration settings. """ - dt_fpath: Path = Path("detections") - gt_fpath: Path = Path("/data/argoverse") - fig_fpath: Path = Path("figs") + dt_fpath: Path = MISSING + gt_fpath: Path = MISSING + fig_fpath: Path = MISSING dt_cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: From d621b3c05704a7dc0345300d172686eb971378ba Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 14:56:20 +0000 Subject: [PATCH 019/113] Fixed init variables + method signatures. --- argoverse/evaluation/detection_utils.py | 9 +++++---- argoverse/evaluation/eval_detection.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index b7ed23e1..f8faddcc 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -42,7 +42,7 @@ class FilterMetric(Enum): def filter_instances( - instances: np.ndarray, + instances: List[ObjectLabelRecord], target_class: str, filter_metric: FilterMetric = FilterMetric.EUCLIDEAN, max_dist: float = 50.0, @@ -50,7 +50,7 @@ def filter_instances( """Filter the GT annotations based on a set of conditions (e.g., distance from egovehicle). Args: - instances: The instances to be filtered. + instances: The instances to be filtered (N,). target_class: The name of the class of interest. filter_metric: The range metric used for filtering. max_dist: The maximum distance for range filtering. @@ -67,8 +67,9 @@ def filter_instances( if centers.shape[0] > 0: dt_dists = np.linalg.norm(centers, axis=1) filtered_annos = instances[dt_dists < max_dist] - - return filtered_annos + else: + raise NotImplementedError("This filter metric is not implemented!") + return filtered_annos def get_ranks(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index efe28dd4..497cecd6 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -3,7 +3,7 @@ import logging import os from collections import defaultdict -from dataclasses import MISSING, dataclass, field +from dataclasses import dataclass, field from multiprocessing import Pool from pathlib import Path from typing import DefaultDict, List, Tuple @@ -68,9 +68,9 @@ class DetectionEvaluator: dt_cfg: The detection configuration settings. """ - dt_fpath: Path = MISSING - gt_fpath: Path = MISSING - fig_fpath: Path = MISSING + dt_fpath: Path + gt_fpath: Path + fig_fpath: Path dt_cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: @@ -90,6 +90,7 @@ def evaluate(self) -> pd.DataFrame: with Pool(os.cpu_count()) as p: accum = p.map(self.accumulate, gt_fpaths) + # self.accumulate(gt_fpaths[0]) for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): From e16b853ea3ee659e8f1d5f41be1cd4aa37205d91 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 16:59:56 +0000 Subject: [PATCH 020/113] Removed unused imports, corrected method signatures, added new pre-commit-configs --- .pre-commit-config.yaml | 4 ++++ argoverse/evaluation/detection_utils.py | 11 ++++------- argoverse/evaluation/eval_tracking.py | 14 ++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a11eae3..ec6090be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,4 +19,8 @@ repos: hooks: - id: black args: ["-l", "120"] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: d6e31ae + hooks: + - id: mypy exclude: "docs/.*$" diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index f8faddcc..18c8e4c0 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -96,7 +96,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: method: Accumulation method. Returns: - Interpolated precision at all recall levels. + prec_interp: Interpolated precision at all recall levels. """ if method == InterpType.ALL: prec_interp = np.maximum.accumulate(prec[::-1])[::-1] @@ -164,14 +164,11 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar raise NotImplemented("This distance metric is not implemented!") -def normalize_angle(angle: float) -> float: - """Map angle (in radians) from domain [-π, π] to [0, 2π). - - Args: - angle: Angle (in radians) in domain [-π, π]. +def normalize_angle(angle: np.ndarray) -> np.ndarray: + """Map angle (in radians) from domain [-π, π] to [0, π). Returns: - The angle (in radians) mapped to the interval [0, 2π). + The angle (in radians) mapped to the interval [0, π]. """ period = 2 * np.pi phase_shift = np.pi diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 84d82e85..d288b68e 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -1,24 +1,20 @@ # import argparse import glob -import json import logging import os import pathlib import pickle -import sys from pathlib import Path -from typing import Any, Dict, List, TextIO, Tuple, Union +from typing import Any, Dict, List, TextIO, Union import motmetrics as mm import numpy as np from shapely.geometry.polygon import Polygon -from argoverse.evaluation.eval_utils import get_pc_inside_bbox, label_to_bbox +from argoverse.evaluation.eval_utils import label_to_bbox from argoverse.utils.json_utils import read_json_file -from argoverse.utils.ply_loader import load_ply from argoverse.utils.se3 import SE3 -from argoverse.utils.transform import quat2rotmat mh = mm.metrics.create() logger = logging.getLogger(__name__) @@ -82,7 +78,9 @@ def get_distance_iou_3d(x1: np.ndarray, x2: np.ndarray, name: str = "bbox") -> f return float(score) -def get_orientation_error_deg(yaw1: float, yaw2: float) -> float: +def get_orientation_error_deg( + yaw1: Union[float, np.ndarray], yaw2: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: """ Compute the smallest difference between 2 angles, in magnitude (absolute difference). First, find the difference between the two yaw angles; since @@ -119,7 +117,7 @@ def get_orientation_error_deg(yaw1: float, yaw2: float) -> float: return float(error) -def get_distance(x1: np.ndarray, x2: np.ndarray, name: str) -> float: +def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str) -> float: """Get the distance between two poses, returns nan if distance is larger than detection threshold. Args: From 25ef3cdec296c374737c758fda033d6fd55e89e5 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 19:41:33 +0000 Subject: [PATCH 021/113] Style fixes + function renaming. --- argoverse/evaluation/detection_utils.py | 7 ++++--- argoverse/evaluation/eval_detection.py | 10 +++++----- tests/test_eval_detection.py | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 18c8e4c0..9545bc51 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -81,7 +81,6 @@ def get_ranks(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: Returns: scores: The detection scores (N,). ranks: The ranking for the detections (N,). - """ scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] @@ -96,7 +95,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: method: Accumulation method. Returns: - prec_interp: Interpolated precision at all recall levels. + prec_interp: Interpolated precision at all recall levels (N,). """ if method == InterpType.ALL: prec_interp = np.maximum.accumulate(prec[::-1])[::-1] @@ -105,7 +104,9 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: return prec_interp -def compute_match_matrix(dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: SimFnType) -> np.ndarray: +def compute_affinity_matrix( + dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: SimFnType +) -> np.ndarray: """Calculate the match matrix between detections and ground truth labels, using a specified similarity function. diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 497cecd6..c0ce7717 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -17,7 +17,7 @@ from argoverse.evaluation.detection_utils import ( DistFnType, SimFnType, - compute_match_matrix, + compute_affinity_matrix, dist_fn, filter_instances, get_ranks, @@ -164,20 +164,20 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: if gts.shape[0] == 0: return np.hstack((error_types, scores)) - match_matrix = compute_match_matrix(dts, gts, self.dt_cfg.sim_fn_type) + affinity_matrix = compute_affinity_matrix(dts, gts, self.dt_cfg.sim_fn_type) # Get the most similar GT label for each detection. - gt_matches = np.expand_dims(match_matrix[ranks].argmax(axis=1), axis=0) + gt_matches = np.expand_dims(affinity_matrix[ranks].argmax(axis=1), axis=0) # Grab the corresponding similarity score for each assignment. - match_scores = np.take_along_axis(match_matrix[ranks].T, gt_matches, axis=0).squeeze(0) + affinities = np.take_along_axis(affinity_matrix[ranks].T, gt_matches, axis=0).squeeze(0) # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) for i, thr in enumerate(self.dt_cfg.sim_ths): # tp_mask may need to be defined differently with other similarity metrics - tp_mask = match_scores[unique_dt_matches] > -thr + tp_mask = affinities[unique_dt_matches] > -thr error_types[unique_dt_matches, i] = tp_mask if thr == self.dt_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index fc84d6da..4a798f29 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -10,7 +10,7 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_match_matrix, dist_fn +from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_affinity_matrix, dist_fn from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator from argoverse.utils.transform import quat_scipy2argo_vectorized @@ -36,7 +36,7 @@ def test_center_similarity() -> None: """Test that the Center similarity function works.""" olr1 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) olr2 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) - assert compute_match_matrix(olr1, olr2, SimFnType.CENTER) == -5 + assert compute_affinity_matrix(olr1, olr2, SimFnType.CENTER) == -5 def test_translation_distance() -> None: From 65e36c1710ed5cf4228acb07e57301af17421fa6 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 17 Sep 2020 20:43:49 +0000 Subject: [PATCH 022/113] Docstring cleanup + named magic variables. --- argoverse/evaluation/eval_detection.py | 89 ++++++++++++++------------ 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index c0ce7717..88f4cfdf 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -2,7 +2,7 @@ import argparse import logging import os -from collections import defaultdict +from collections import ChainMap, defaultdict from dataclasses import dataclass, field from multiprocessing import Pool from pathlib import Path @@ -11,6 +11,7 @@ import matplotlib import numpy as np import pandas as pd +from pandas.core import frame from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT from argoverse.data_loading.object_label_record import read_label @@ -33,22 +34,27 @@ logger = logging.getLogger(__name__) +METRIC_NAMES: List[str] = ["AP", "ATE", "ASE", "AOE", "CDS"] +METRIC_DEFAULT_VALUES: List[float] = [0, 1.0, 1.0, 1.0, 0] + + @dataclass class DetectionCfg: """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. Args: - sim_ths: The similarity thresholds for determining a true positive. - sim_fn_type: The type of similarity function to be used for calculating - average precision. + affinity_threshs: The affinity thresholds for determining a true positive. + affinity_fn_type: The type of affinity function to be used for calculating average precision. n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. - dcs: The weight vector for the detection composite score (DCS). - tp_threshold: Center distance threshold for the true positive metrics (in meters). + cds_weights: The weight vector for the detection composite score (CDS). + tp_thresh: Center distance threshold for the true positive metrics (in meters). significant_digits: The precision for metrics. + detection_classes: Detection classes for evaluation. + save_figs: Flag to save figures. """ - sim_ths: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) - sim_fn_type: SimFnType = SimFnType.CENTER + affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) + affinity_fn_type: SimFnType = SimFnType.CENTER n_rec_samples: int = 101 cds_weights: List[float] = field(default_factory=lambda: [3, 1, 1, 1]) tp_thresh: float = 2.0 @@ -62,16 +68,16 @@ class DetectionEvaluator: """Instantiates a DetectionEvaluator object for evaluation. Args: - dt_fpath: The path to the folder which contains the detections. - gt_fpath: The path to the folder which contains all the logs. - fig_fpath: The path to the folder which will contain the output figures. - dt_cfg: The detection configuration settings. + detection_fpath: The path to the folder which contains the detections. + ground_truth_fpath: The path to the folder which contains all the logs. + figures_fpath: The path to the folder which will contain the output figures. + detection_cfg: The detection configuration settings. """ - dt_fpath: Path - gt_fpath: Path - fig_fpath: Path - dt_cfg: DetectionCfg = DetectionCfg() + detection_fpath: Path + ground_truth_fpath: Path + figures_fpath: Path + detection_cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: """Evaluate detection output and return metrics. The multiprocessing @@ -79,10 +85,11 @@ def evaluate(self) -> pd.DataFrame: annotations. Returns: - The evaluation metrics. + Evaluation metrics of shape (C + 1, K) where C + 1 is the number of classes + plus a row for their means. K refers to the number of evaluation metrics. """ - dt_fpaths = list(self.dt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) - gt_fpaths = list(self.gt_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + dt_fpaths = list(self.detection_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + gt_fpaths = list(self.ground_truth_fpath.glob("*/per_sweep_annotations_amodal/*.json")) assert len(dt_fpaths) == len(gt_fpaths) data: DefaultDict[str, np.ndarray] = defaultdict(list) @@ -90,7 +97,6 @@ def evaluate(self) -> pd.DataFrame: with Pool(os.cpu_count()) as p: accum = p.map(self.accumulate, gt_fpaths) - # self.accumulate(gt_fpaths[0]) for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): @@ -100,18 +106,17 @@ def evaluate(self) -> pd.DataFrame: data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) - columns = ["AP", "ATE", "ASE", "AOE", "CDS"] - - # Initialize metrics table [0, 1.0, 1.0, 1.0, 0] represents the default value for each column. - init_data = {k: [0, 1.0, 1.0, 1.0, 0] for k in self.dt_cfg.detection_classes} - summary = pd.DataFrame.from_dict(init_data, orient="index", columns=columns) - summary_update = pd.DataFrame.from_dict(self.summarize(data, cls_to_ninst), orient="index", columns=columns) + init_data = {k: METRIC_DEFAULT_VALUES for k in self.detection_cfg.detection_classes} + summary = pd.DataFrame.from_dict(init_data, orient="index", columns=METRIC_NAMES) + summary_update = pd.DataFrame.from_dict( + self.summarize(data, cls_to_ninst), orient="index", columns=METRIC_NAMES + ) summary.update(summary_update) - summary = summary.round(self.dt_cfg.significant_digits) + summary = summary.round(self.detection_cfg.significant_digits) summary.index = summary.index.str.title() - summary.loc["Average Metrics"] = summary.mean().round(self.dt_cfg.significant_digits) + summary.loc["Average Metrics"] = summary.mean().round(self.detection_cfg.significant_digits) return summary def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: @@ -129,14 +134,14 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa logger.info(f"log_id = {log_id}") ts = gt_fpath.stem.split("_")[-1] - dt_fpath = self.dt_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" + dt_fpath = self.detection_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" dts = np.array(read_label(str(dt_fpath))) gts = np.array(read_label(str(gt_fpath))) cls_stats = defaultdict(list) cls_to_ninst = defaultdict(int) - for dt_cls in self.dt_cfg.detection_classes: + for dt_cls in self.detection_cfg.detection_classes: dt_filtered = filter_instances(dts, dt_cls) gt_filtered = filter_instances(gts, dt_cls) @@ -158,13 +163,13 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: Returns: True positives, false positives, scores, and translation errors. """ - n_threshs = len(self.dt_cfg.sim_ths) + n_threshs = len(self.detection_cfg.affinity_threshs) error_types = np.zeros((dts.shape[0], n_threshs + 3)) scores, ranks = get_ranks(dts) if gts.shape[0] == 0: return np.hstack((error_types, scores)) - affinity_matrix = compute_affinity_matrix(dts, gts, self.dt_cfg.sim_fn_type) + affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) # Get the most similar GT label for each detection. gt_matches = np.expand_dims(affinity_matrix[ranks].argmax(axis=1), axis=0) @@ -174,13 +179,13 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) - for i, thr in enumerate(self.dt_cfg.sim_ths): + for i, thr in enumerate(self.detection_cfg.affinity_threshs): # tp_mask may need to be defined differently with other similarity metrics tp_mask = affinities[unique_dt_matches] > -thr error_types[unique_dt_matches, i] = tp_mask - if thr == self.dt_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: + if thr == self.detection_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] @@ -210,17 +215,17 @@ def summarize( summary: The summary statistics. """ summary: DefaultDict[str, List] = defaultdict(list) - recalls_interp = np.linspace(0, 1, self.dt_cfg.n_rec_samples) - num_ths = len(self.dt_cfg.sim_ths) - if not self.fig_fpath.is_dir(): - self.fig_fpath.mkdir(parents=True, exist_ok=True) + recalls_interp = np.linspace(0, 1, self.detection_cfg.n_rec_samples) + num_ths = len(self.detection_cfg.affinity_threshs) + if not self.figures_fpath.is_dir(): + self.figures_fpath.mkdir(parents=True, exist_ok=True) for cls_name, cls_stats in data.items(): ninst = cls_to_ninst[cls_name] ranks = cls_stats[:, -1].argsort()[::-1] cls_stats = cls_stats[ranks] - for i, _ in enumerate(self.dt_cfg.sim_ths): + for i, _ in enumerate(self.detection_cfg.affinity_threshs): tp = cls_stats[:, i].astype(bool) cumulative_tp = np.cumsum(tp, dtype=np.int) @@ -237,7 +242,7 @@ def summarize( ap_th = prec_truncated.mean() summary[cls_name] += [ap_th] - if self.dt_cfg.save_figs: + if self.detection_cfg.save_figs: self.plot(recalls_interp, precisions_interp, cls_name) # AP Metric @@ -251,7 +256,7 @@ def summarize( cds_summands = np.hstack((ap, np.clip(1 - tp_metrics, a_min=0, a_max=None))) # Ranking metric - cds = np.average(cds_summands, weights=self.dt_cfg.cds_weights) + cds = np.average(cds_summands, weights=self.detection_cfg.cds_weights) summary[cls_name] = [ap, *tp_metrics, cds] @@ -270,7 +275,7 @@ def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) - plt.title("PR Curve") plt.xlabel("Recall") plt.ylabel("Precision") - plt.savefig(f"{self.fig_fpath}/{cls_name}.png") + plt.savefig(f"{self.figures_fpath}/{cls_name}.png") plt.close() From 73fbc829f01c63d45862cd64045f84c920670075 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 00:31:29 +0000 Subject: [PATCH 023/113] Removed truncation and cleaned up docstrings. --- argoverse/evaluation/eval_detection.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 88f4cfdf..6d9ae1ca 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -2,7 +2,7 @@ import argparse import logging import os -from collections import ChainMap, defaultdict +from collections import defaultdict from dataclasses import dataclass, field from multiprocessing import Pool from pathlib import Path @@ -34,7 +34,8 @@ logger = logging.getLogger(__name__) -METRIC_NAMES: List[str] = ["AP", "ATE", "ASE", "AOE", "CDS"] +TP_METRIC_NAMES = ["ATE", "ASE", "AOE"] +METRIC_NAMES: List[str] = ["AP"] + TP_METRIC_NAMES + ["CDS"] METRIC_DEFAULT_VALUES: List[float] = [0, 1.0, 1.0, 1.0, 0] @@ -148,12 +149,13 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa logger.info(f"{dt_filtered.shape[0]} detections") logger.info(f"{gt_filtered.shape[0]} ground truth") if dt_filtered.shape[0] > 0: - cls_stats[dt_cls] = self.assign(dt_filtered, gt_filtered) + metrics, scores = self.assign(dt_filtered, gt_filtered) + cls_stats[dt_cls] = np.hstack((metrics, scores)) cls_to_ninst[dt_cls] = gt_filtered.shape[0] return cls_stats, cls_to_ninst - def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: + def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: """Attempt assignment of each detection to a ground truth label. Args: @@ -161,13 +163,14 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: gts: Ground truth labels of shape (M,). Returns: - True positives, false positives, scores, and translation errors. + metrics: Flags indicating true/false positive and true positive errors (N, 8). + scores: Corresponding scores for the true positives/false positives (N,) """ n_threshs = len(self.detection_cfg.affinity_threshs) - error_types = np.zeros((dts.shape[0], n_threshs + 3)) + metrics = np.zeros((dts.shape[0], n_threshs + len(TP_METRIC_NAMES))) scores, ranks = get_ranks(dts) if gts.shape[0] == 0: - return np.hstack((error_types, scores)) + return np.hstack((metrics, scores)) affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) @@ -183,7 +186,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: # tp_mask may need to be defined differently with other similarity metrics tp_mask = affinities[unique_dt_matches] > -thr - error_types[unique_dt_matches, i] = tp_mask + metrics[unique_dt_matches, i] = tp_mask if thr == self.detection_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: dt_tp_indices = unique_dt_matches[tp_mask] @@ -196,11 +199,11 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> np.ndarray: scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) - error_types[dt_tp_indices, n_threshs : n_threshs + 3] = np.vstack( + metrics[dt_tp_indices, n_threshs : n_threshs + 3] = np.vstack( (trans_error, scale_error, orient_error) ).T - return np.hstack((error_types, scores)) + return metrics, scores def summarize( self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] @@ -238,8 +241,7 @@ def summarize( precisions = interp(precisions) precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) - prec_truncated = precisions_interp[10 + 1 :] - ap_th = prec_truncated.mean() + ap_th = precisions_interp.mean() summary[cls_name] += [ap_th] if self.detection_cfg.save_figs: From d1c4799b5e7a346694baa2c0e4bf917d2eecfa98 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 01:26:29 +0000 Subject: [PATCH 024/113] Return fix. --- argoverse/evaluation/eval_detection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 6d9ae1ca..803f7335 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -170,7 +170,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr metrics = np.zeros((dts.shape[0], n_threshs + len(TP_METRIC_NAMES))) scores, ranks = get_ranks(dts) if gts.shape[0] == 0: - return np.hstack((metrics, scores)) + return metrics, scores affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) @@ -202,7 +202,6 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr metrics[dt_tp_indices, n_threshs : n_threshs + 3] = np.vstack( (trans_error, scale_error, orient_error) ).T - return metrics, scores def summarize( From e5e7f394b28d266ef2b9c8277b19b2595cb414cb Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 02:28:58 +0000 Subject: [PATCH 025/113] Fixed tp metric defaults + additional constants. --- argoverse/evaluation/eval_detection.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 803f7335..ebb238a8 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -35,8 +35,10 @@ TP_METRIC_NAMES = ["ATE", "ASE", "AOE"] +NUM_TP_METRICS = len(TP_METRIC_NAMES) + METRIC_NAMES: List[str] = ["AP"] + TP_METRIC_NAMES + ["CDS"] -METRIC_DEFAULT_VALUES: List[float] = [0, 1.0, 1.0, 1.0, 0] +METRIC_DEFAULT_VALUES: List[float] = [0.0, 1.0, 1.0, 1.0, 0.0] @dataclass @@ -107,7 +109,7 @@ def evaluate(self) -> pd.DataFrame: data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) - init_data = {k: METRIC_DEFAULT_VALUES for k in self.detection_cfg.detection_classes} + init_data = {dt_cls: METRIC_DEFAULT_VALUES for dt_cls in self.detection_cfg.detection_classes} summary = pd.DataFrame.from_dict(init_data, orient="index", columns=METRIC_NAMES) summary_update = pd.DataFrame.from_dict( self.summarize(data, cls_to_ninst), orient="index", columns=METRIC_NAMES @@ -167,7 +169,8 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr scores: Corresponding scores for the true positives/false positives (N,) """ n_threshs = len(self.detection_cfg.affinity_threshs) - metrics = np.zeros((dts.shape[0], n_threshs + len(TP_METRIC_NAMES))) + metrics = np.zeros((dts.shape[0], n_threshs + NUM_TP_METRICS)) + metrics[:, n_threshs : n_threshs + NUM_TP_METRICS] = 1 scores, ranks = get_ranks(dts) if gts.shape[0] == 0: return metrics, scores @@ -199,7 +202,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) - metrics[dt_tp_indices, n_threshs : n_threshs + 3] = np.vstack( + metrics[dt_tp_indices, n_threshs : n_threshs + NUM_TP_METRICS] = np.vstack( (trans_error, scale_error, orient_error) ).T return metrics, scores @@ -250,7 +253,7 @@ def summarize( ap = np.array(summary[cls_name][:num_ths]).mean() # TP Error Metrics - tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + 3], axis=0) + tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS], axis=0) tp_metrics[2] /= np.pi / 2 # normalize orientation # clip so that we don't get negative values in (1 - ATE) From acc703cafc8161e78057729663ebea8dbb957e94 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 03:29:27 +0000 Subject: [PATCH 026/113] Corrected angle normalization, added new CDS calculation. --- argoverse/evaluation/eval_detection.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index ebb238a8..e55391fc 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -170,7 +170,9 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr """ n_threshs = len(self.detection_cfg.affinity_threshs) metrics = np.zeros((dts.shape[0], n_threshs + NUM_TP_METRICS)) - metrics[:, n_threshs : n_threshs + NUM_TP_METRICS] = 1 + + # Set the true positive metrics to np.nan since error is undefined on false positives. + metrics[:, n_threshs : n_threshs + NUM_TP_METRICS] = np.nan scores, ranks = get_ranks(dts) if gts.shape[0] == 0: return metrics, scores @@ -252,15 +254,22 @@ def summarize( # AP Metric ap = np.array(summary[cls_name][:num_ths]).mean() - # TP Error Metrics - tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS], axis=0) - tp_metrics[2] /= np.pi / 2 # normalize orientation + # Select only the true positives for each instance. + tp_metrics_mask = ~np.isnan(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS]).all(axis=1) + + # If there are no true positives set errors to 1 (and orientation to np.pi / 2 due to normalization below) + # TODO We might consider normalizing the translation error with a max detection distance. + if ~tp_metrics_mask.any(): + tp_metrics = np.array([1.0, 1.0, np.pi]) + else: + # Calculate TP metrics. + tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS][tp_metrics_mask], axis=0) - # clip so that we don't get negative values in (1 - ATE) - cds_summands = np.hstack((ap, np.clip(1 - tp_metrics, a_min=0, a_max=None))) + # Normalize orientation. + tp_metrics[2] /= np.pi - # Ranking metric - cds = np.average(cds_summands, weights=self.detection_cfg.cds_weights) + # Ranking metric (AP * (1 - TP_METRICS)). Clipped to ensure >= 0. + cds = ap * np.clip(1 - tp_metrics, a_min=0, a_max=None).sum() summary[cls_name] = [ap, *tp_metrics, cds] From 8c8d71dfec6a62a96582c10e54f20d6b21c54022 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 18:19:00 +0000 Subject: [PATCH 027/113] Updated CDS calculation + configurable range/metric. --- argoverse/evaluation/detection_utils.py | 9 ++---- argoverse/evaluation/eval_detection.py | 37 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 9545bc51..6c7fd008 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -42,10 +42,7 @@ class FilterMetric(Enum): def filter_instances( - instances: List[ObjectLabelRecord], - target_class: str, - filter_metric: FilterMetric = FilterMetric.EUCLIDEAN, - max_dist: float = 50.0, + instances: List[ObjectLabelRecord], target_class: str, filter_metric: FilterMetric, max_detection_range: float ) -> np.ndarray: """Filter the GT annotations based on a set of conditions (e.g., distance from egovehicle). @@ -53,7 +50,7 @@ def filter_instances( instances: The instances to be filtered (N,). target_class: The name of the class of interest. filter_metric: The range metric used for filtering. - max_dist: The maximum distance for range filtering. + max_detection_range: The maximum distance for range filtering. Returns: The filtered annotations. @@ -66,7 +63,7 @@ def filter_instances( if centers.shape[0] > 0: dt_dists = np.linalg.norm(centers, axis=1) - filtered_annos = instances[dt_dists < max_dist] + filtered_annos = instances[dt_dists < max_detection_range] else: raise NotImplementedError("This filter metric is not implemented!") return filtered_annos diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index e55391fc..95a30e93 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -3,7 +3,7 @@ import logging import os from collections import defaultdict -from dataclasses import dataclass, field +from dataclasses import Field, dataclass, field from multiprocessing import Pool from pathlib import Path from typing import DefaultDict, List, Tuple @@ -17,6 +17,7 @@ from argoverse.data_loading.object_label_record import read_label from argoverse.evaluation.detection_utils import ( DistFnType, + FilterMetric, SimFnType, compute_affinity_matrix, dist_fn, @@ -49,21 +50,27 @@ class DetectionCfg: affinity_threshs: The affinity thresholds for determining a true positive. affinity_fn_type: The type of affinity function to be used for calculating average precision. n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. - cds_weights: The weight vector for the detection composite score (CDS). tp_thresh: Center distance threshold for the true positive metrics (in meters). significant_digits: The precision for metrics. detection_classes: Detection classes for evaluation. + detection_metric: The detection metric to use for filtering of both detections and ground truth annotations. + max_detection_range: The max distance (under a specific metric) for a a detection or ground truth to be considered for evaluation. save_figs: Flag to save figures. """ affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) affinity_fn_type: SimFnType = SimFnType.CENTER n_rec_samples: int = 101 - cds_weights: List[float] = field(default_factory=lambda: [3, 1, 1, 1]) tp_thresh: float = 2.0 significant_digits: int = 3 detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) + detection_metric: FilterMetric = FilterMetric.EUCLIDEAN + max_detection_range: float = 50.0 save_figs: bool = False + tp_normalization_terms: np.ndarray = field(init=False) + + def __post_init__(self): + self.tp_normalization_terms: np.ndarray = np.array([self.tp_thresh, 1.0, np.pi]) @dataclass @@ -145,8 +152,18 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa cls_stats = defaultdict(list) cls_to_ninst = defaultdict(int) for dt_cls in self.detection_cfg.detection_classes: - dt_filtered = filter_instances(dts, dt_cls) - gt_filtered = filter_instances(gts, dt_cls) + dt_filtered = filter_instances( + dts, + dt_cls, + filter_metric=self.detection_cfg.detection_metric, + max_detection_range=self.detection_cfg.max_detection_range, + ) + gt_filtered = filter_instances( + gts, + dt_cls, + filter_metric=self.detection_cfg.detection_metric, + max_detection_range=self.detection_cfg.max_detection_range, + ) logger.info(f"{dt_filtered.shape[0]} detections") logger.info(f"{gt_filtered.shape[0]} ground truth") @@ -260,16 +277,16 @@ def summarize( # If there are no true positives set errors to 1 (and orientation to np.pi / 2 due to normalization below) # TODO We might consider normalizing the translation error with a max detection distance. if ~tp_metrics_mask.any(): - tp_metrics = np.array([1.0, 1.0, np.pi]) + tp_metrics = self.detection_cfg.tp_normalization_terms else: # Calculate TP metrics. tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS][tp_metrics_mask], axis=0) - # Normalize orientation. - tp_metrics[2] /= np.pi + # Convert errors to scores + tp_scores = 1 - (tp_metrics / self.detection_cfg.tp_normalization_terms) - # Ranking metric (AP * (1 - TP_METRICS)). Clipped to ensure >= 0. - cds = ap * np.clip(1 - tp_metrics, a_min=0, a_max=None).sum() + # Compute Composite Detection Score (CDS) + cds = ap * tp_scores.mean() summary[cls_name] = [ap, *tp_metrics, cds] From 6e2d7545f28b8d9bfa5e443c2a12c466a43f93b1 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Sep 2020 18:21:40 +0000 Subject: [PATCH 028/113] Fixed comments. --- argoverse/evaluation/eval_detection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 95a30e93..089500ed 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -274,8 +274,7 @@ def summarize( # Select only the true positives for each instance. tp_metrics_mask = ~np.isnan(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS]).all(axis=1) - # If there are no true positives set errors to 1 (and orientation to np.pi / 2 due to normalization below) - # TODO We might consider normalizing the translation error with a max detection distance. + # If there are no true positives set tp errors to their maximum values due to normalization below) if ~tp_metrics_mask.any(): tp_metrics = self.detection_cfg.tp_normalization_terms else: From f2131818ba27e668d0eeb3656ec4dfb1d7410955 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Mon, 21 Sep 2020 10:51:04 -0400 Subject: [PATCH 029/113] Changed variable names, cleaned up docstrings, changed default evaluation range. --- argoverse/evaluation/detection_utils.py | 12 +-- argoverse/evaluation/eval_detection.py | 107 +++++++++++++++++++----- 2 files changed, 90 insertions(+), 29 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 6c7fd008..9b825802 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -42,20 +42,20 @@ class FilterMetric(Enum): def filter_instances( - instances: List[ObjectLabelRecord], target_class: str, filter_metric: FilterMetric, max_detection_range: float + instances: List[ObjectLabelRecord], target_class_name: str, filter_metric: FilterMetric, max_detection_range: float ) -> np.ndarray: - """Filter the GT annotations based on a set of conditions (e.g., distance from egovehicle). + """Filter the GT annotations based on a set of conditions (classname and distance from egovehicle). Args: instances: The instances to be filtered (N,). - target_class: The name of the class of interest. + target_class_name: The name of the class of interest. filter_metric: The range metric used for filtering. max_detection_range: The maximum distance for range filtering. Returns: The filtered annotations. """ - instances = np.array([instance for instance in instances if instance.label_class == target_class]) + instances = np.array([instance for instance in instances if instance.label_class == target_class_name]) if filter_metric == FilterMetric.EUCLIDEAN: centers = np.array([dt.translation for dt in instances]) @@ -128,8 +128,8 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar """Distance functions between detections and ground truth. Args: - dts: Detections. - gts: Ground truth labels. + dts: Detections (N,). + gts: Ground truth labels (M,). metric: Distance function type. Returns: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 089500ed..4cdef006 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -1,4 +1,58 @@ # +"""Detection evaluation for the Argoverse detection leaderboard. + +Evaluation: + + Precision/Recall + + 1. Average Precision: Standard VOC-style average precision calculation + except a true positive requires a bird's eye center distance of less + than a predefined threshold. + + True Positive Errors + + All true positive errors -- as the name implies -- accumulate error solely + when an object is a true positive match to a ground truth detection. The matching + criterion is represented by `tp_thresh` in the DetectionCfg class. + + 1. Average Translation Error: The average Euclidean distance (center-based) between a + detection and its ground truth assignment. + + 2. Average Scale Error: The average intersection over union (IoU) after the prediction + and assigned ground truth's pose has been aligned. + + 3. Average Orientation Error: The average angular distance between the detection and + the assigned ground truth. We choose the smallest angle between the two different + headings when calculating the error. + + Composite Scores + + 1. Composite Detection Score: The ranking metric for the detection leaderboard. This + is computed as the product of mAP with the sum of the complements of the true positive + errors (after normalization), i.e., + + Average Translation Measure (ATM): ATE / TP_THRESHOLD; 0 <= 1 - ATE / TP_THRESHOLD <= 1 + Average Scaling Measure (ASM): 1 - ASE / 1; 0 <= 1 - ASE / 1 <= 1 + Average Orientation Measure (AOM): 1 - AOE / PI; 0 <= 1 - AOE / PI <= 1 + + These (as well as AP) are averaged over each detection class to produce: + + mAP, mATM, mASM, mAOM. + + Lastly, the Composite Detection Score is computed as: + + CDS = mAP * (mATE + mASE + mAOE); 0 <= mAP * (mATE + mASE + mAOE) <= 1. + + ** In the case of no true positives under the specified threshold, the true positive measures + will assume their upper bounds of 1.0. respectively. + +Results: + + The results are represented as a (C + 1, P) table, where C + 1 represents the number of evaluation classes + in addition to the mean statistics average across all classes, and P refers to the number of included statistics, + e.g. AP, ATE, ASE, AOE, CDS by default. + +""" import argparse import logging import os @@ -35,11 +89,14 @@ logger = logging.getLogger(__name__) -TP_METRIC_NAMES = ["ATE", "ASE", "AOE"] -NUM_TP_METRICS = len(TP_METRIC_NAMES) +TP_ERROR_NAMES = ["ATE", "ASE", "AOE"] +N_TP_ERRORS = len(TP_ERROR_NAMES) + +STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] + +MEASURE_DEFAULT_VALUES: List[float] = [0.0, 1.0, 1.0, 1.0, 0.0] -METRIC_NAMES: List[str] = ["AP"] + TP_METRIC_NAMES + ["CDS"] -METRIC_DEFAULT_VALUES: List[float] = [0.0, 1.0, 1.0, 1.0, 0.0] +MAX_YAW_ERROR = np.pi @dataclass @@ -65,12 +122,12 @@ class DetectionCfg: significant_digits: int = 3 detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) detection_metric: FilterMetric = FilterMetric.EUCLIDEAN - max_detection_range: float = 50.0 + max_detection_range: float = 100.0 save_figs: bool = False tp_normalization_terms: np.ndarray = field(init=False) def __post_init__(self): - self.tp_normalization_terms: np.ndarray = np.array([self.tp_thresh, 1.0, np.pi]) + self.tp_normalization_terms: np.ndarray = np.array([self.tp_thresh, 1.0, MAX_YAW_ERROR]) @dataclass @@ -116,10 +173,10 @@ def evaluate(self) -> pd.DataFrame: data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) - init_data = {dt_cls: METRIC_DEFAULT_VALUES for dt_cls in self.detection_cfg.detection_classes} - summary = pd.DataFrame.from_dict(init_data, orient="index", columns=METRIC_NAMES) + init_data = {dt_cls: MEASURE_DEFAULT_VALUES for dt_cls in self.detection_cfg.detection_classes} + summary = pd.DataFrame.from_dict(init_data, orient="index", columns=STATISTIC_NAMES) summary_update = pd.DataFrame.from_dict( - self.summarize(data, cls_to_ninst), orient="index", columns=METRIC_NAMES + self.summarize(data, cls_to_ninst), orient="index", columns=STATISTIC_NAMES ) summary.update(summary_update) @@ -136,7 +193,8 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa gt_fpath: Ground truth file path. Returns: - cls_stats: Class statistics of shape (N, 8) + cls_stats: Class statistics of shape ((N, K + S) where K is the number of true positive thresholds used + for AP computation and S is the number of true positive errors. cls_to_ninst: Mapping of the class names to the number of instances in the ground truth dataset. """ @@ -151,16 +209,16 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa cls_stats = defaultdict(list) cls_to_ninst = defaultdict(int) - for dt_cls in self.detection_cfg.detection_classes: + for class_name in self.detection_cfg.detection_classes: dt_filtered = filter_instances( dts, - dt_cls, + class_name, filter_metric=self.detection_cfg.detection_metric, max_detection_range=self.detection_cfg.max_detection_range, ) gt_filtered = filter_instances( gts, - dt_cls, + class_name, filter_metric=self.detection_cfg.detection_metric, max_detection_range=self.detection_cfg.max_detection_range, ) @@ -169,9 +227,9 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa logger.info(f"{gt_filtered.shape[0]} ground truth") if dt_filtered.shape[0] > 0: metrics, scores = self.assign(dt_filtered, gt_filtered) - cls_stats[dt_cls] = np.hstack((metrics, scores)) + cls_stats[class_name] = np.hstack((metrics, scores)) - cls_to_ninst[dt_cls] = gt_filtered.shape[0] + cls_to_ninst[class_name] = gt_filtered.shape[0] return cls_stats, cls_to_ninst def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: @@ -182,14 +240,15 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr gts: Ground truth labels of shape (M,). Returns: - metrics: Flags indicating true/false positive and true positive errors (N, 8). + metrics: Matrix of true/false positive concatenated with true positive errors (N, K + S) where K is the number + of true positive thresholds used for AP computation and S is the number of true positive errors. scores: Corresponding scores for the true positives/false positives (N,) """ n_threshs = len(self.detection_cfg.affinity_threshs) - metrics = np.zeros((dts.shape[0], n_threshs + NUM_TP_METRICS)) + metrics = np.zeros((dts.shape[0], n_threshs + N_TP_ERRORS)) # Set the true positive metrics to np.nan since error is undefined on false positives. - metrics[:, n_threshs : n_threshs + NUM_TP_METRICS] = np.nan + metrics[:, n_threshs : n_threshs + N_TP_ERRORS] = np.nan scores, ranks = get_ranks(dts) if gts.shape[0] == 0: return metrics, scores @@ -197,9 +256,11 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) # Get the most similar GT label for each detection. - gt_matches = np.expand_dims(affinity_matrix[ranks].argmax(axis=1), axis=0) + gt_matches = affinity_matrix[ranks].argmax(axis=1)[np.newaxis, :] - # Grab the corresponding similarity score for each assignment. + # The affinity matrix is an N by M matrix of the detections and ground truth labels respectively. + # We want to take the corresponding affinity for each of the initial assignments using `gt_matches`. + # The following line grabs the max affinity for each detection to a ground truth label. affinities = np.take_along_axis(affinity_matrix[ranks].T, gt_matches, axis=0).squeeze(0) # Find the indices of the "first" detection assigned to each GT. @@ -221,7 +282,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) - metrics[dt_tp_indices, n_threshs : n_threshs + NUM_TP_METRICS] = np.vstack( + metrics[dt_tp_indices, n_threshs : n_threshs + N_TP_ERRORS] = np.vstack( (trans_error, scale_error, orient_error) ).T return metrics, scores @@ -272,14 +333,14 @@ def summarize( ap = np.array(summary[cls_name][:num_ths]).mean() # Select only the true positives for each instance. - tp_metrics_mask = ~np.isnan(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS]).all(axis=1) + tp_metrics_mask = ~np.isnan(cls_stats[:, num_ths : num_ths + N_TP_ERRORS]).all(axis=1) # If there are no true positives set tp errors to their maximum values due to normalization below) if ~tp_metrics_mask.any(): tp_metrics = self.detection_cfg.tp_normalization_terms else: # Calculate TP metrics. - tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + NUM_TP_METRICS][tp_metrics_mask], axis=0) + tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + N_TP_ERRORS][tp_metrics_mask], axis=0) # Convert errors to scores tp_scores = 1 - (tp_metrics / self.detection_cfg.tp_normalization_terms) From d82ea5d7a3148cd143378d2a59963a6b9a34847e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 21 Sep 2020 18:54:28 +0000 Subject: [PATCH 030/113] Ensured fix number of boxes per scene per class. Global variable updates. --- argoverse/evaluation/detection_utils.py | 9 +++--- argoverse/evaluation/eval_detection.py | 38 ++++++++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 9b825802..968851fe 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -44,7 +44,7 @@ class FilterMetric(Enum): def filter_instances( instances: List[ObjectLabelRecord], target_class_name: str, filter_metric: FilterMetric, max_detection_range: float ) -> np.ndarray: - """Filter the GT annotations based on a set of conditions (classname and distance from egovehicle). + """Filter the GT annotations based on a set of conditions (class name and distance from egovehicle). Args: instances: The instances to be filtered (N,). @@ -69,19 +69,20 @@ def filter_instances( return filtered_annos -def get_ranks(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: +def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: """Get the rankings for the detections. Args: dts: Detections (N,). Returns: - scores: The detection scores (N,). ranks: The ranking for the detections (N,). + scores: The detection scores (N,). """ scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] - return np.expand_dims(scores, 1)[ranks], ranks + ranked_detections = dts[ranks] + return ranked_detections, scores[:, np.newaxis] def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 4cdef006..ae257b60 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -76,8 +76,8 @@ compute_affinity_matrix, dist_fn, filter_instances, - get_ranks, interp, + rank, ) matplotlib.use("Agg") # isort:skip @@ -89,15 +89,22 @@ logger = logging.getLogger(__name__) -TP_ERROR_NAMES = ["ATE", "ASE", "AOE"] -N_TP_ERRORS = len(TP_ERROR_NAMES) +TP_ERROR_NAMES: List[str] = ["ATE", "ASE", "AOE"] +N_TP_ERRORS: int = len(TP_ERROR_NAMES) STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] -MEASURE_DEFAULT_VALUES: List[float] = [0.0, 1.0, 1.0, 1.0, 0.0] - MAX_YAW_ERROR = np.pi +MIN_AP: float = 0.0 +MAX_ATM: float = 1.0 +MAX_ASM: float = 1.0 +MAX_AOM: float = 1.0 +MIN_CDS: float = 0.0 +MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_ATM, MAX_ASM, MAX_AOM, MIN_CDS] + +MAX_NUM_BOXES: int = 500 + @dataclass class DetectionCfg: @@ -196,7 +203,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa cls_stats: Class statistics of shape ((N, K + S) where K is the number of true positive thresholds used for AP computation and S is the number of true positive errors. cls_to_ninst: Mapping of the class names to the number of instances in the ground - truth dataset. + truth dataset. """ log_id = gt_fpath.parents[1].stem logger.info(f"log_id = {log_id}") @@ -226,7 +233,8 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa logger.info(f"{dt_filtered.shape[0]} detections") logger.info(f"{gt_filtered.shape[0]} ground truth") if dt_filtered.shape[0] > 0: - metrics, scores = self.assign(dt_filtered, gt_filtered) + ranked_detections, scores = rank(dt_filtered) + metrics = self.assign(ranked_detections, gt_filtered) cls_stats[class_name] = np.hstack((metrics, scores)) cls_to_ninst[class_name] = gt_filtered.shape[0] @@ -244,24 +252,28 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr of true positive thresholds used for AP computation and S is the number of true positive errors. scores: Corresponding scores for the true positives/false positives (N,) """ + + # Ensure the number of boxes considered per class is at most `MAX_NUM_BOXES`. + if dts.shape[0] > MAX_NUM_BOXES: + dts = dts[:MAX_NUM_BOXES] + n_threshs = len(self.detection_cfg.affinity_threshs) metrics = np.zeros((dts.shape[0], n_threshs + N_TP_ERRORS)) # Set the true positive metrics to np.nan since error is undefined on false positives. metrics[:, n_threshs : n_threshs + N_TP_ERRORS] = np.nan - scores, ranks = get_ranks(dts) if gts.shape[0] == 0: - return metrics, scores + return metrics affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) # Get the most similar GT label for each detection. - gt_matches = affinity_matrix[ranks].argmax(axis=1)[np.newaxis, :] + gt_matches = affinity_matrix.argmax(axis=1)[np.newaxis, :] # The affinity matrix is an N by M matrix of the detections and ground truth labels respectively. # We want to take the corresponding affinity for each of the initial assignments using `gt_matches`. # The following line grabs the max affinity for each detection to a ground truth label. - affinities = np.take_along_axis(affinity_matrix[ranks].T, gt_matches, axis=0).squeeze(0) + affinities = np.take_along_axis(affinity_matrix.T, gt_matches, axis=0).squeeze(0) # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) @@ -275,7 +287,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] - dt_df = pd.DataFrame([dt.__dict__ for dt in dts[ranks][dt_tp_indices]]) + dt_df = pd.DataFrame([dt.__dict__ for dt in dts[dt_tp_indices]]) gt_df = pd.DataFrame([gt.__dict__ for gt in gts[gt_tp_indices]]) trans_error = dist_fn(dt_df, gt_df, DistFnType.TRANSLATION) @@ -285,7 +297,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr metrics[dt_tp_indices, n_threshs : n_threshs + N_TP_ERRORS] = np.vstack( (trans_error, scale_error, orient_error) ).T - return metrics, scores + return metrics def summarize( self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] From 4508687e7992685d8726f25d1ed3631180660a19 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 21 Sep 2020 18:57:19 +0000 Subject: [PATCH 031/113] Fixed constants to reflect actual quantity. --- argoverse/evaluation/eval_detection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index ae257b60..75b0b9a4 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -97,11 +97,11 @@ MAX_YAW_ERROR = np.pi MIN_AP: float = 0.0 -MAX_ATM: float = 1.0 -MAX_ASM: float = 1.0 -MAX_AOM: float = 1.0 +MAX_NORM_ATE: float = 1.0 +MAX_NORM_ASE: float = 1.0 +MAX_NORM_AOE: float = 1.0 MIN_CDS: float = 0.0 -MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_ATM, MAX_ASM, MAX_AOM, MIN_CDS] +MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORM_ATE, MAX_NORM_ASE, MAX_NORM_AOE, MIN_CDS] MAX_NUM_BOXES: int = 500 From e44b806754caa4397a22803c72dbfbf0a856777a Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 21 Sep 2020 19:46:33 +0000 Subject: [PATCH 032/113] Formatted json. --- tests/test_data/detection/1/city_info.json | 4 +- .../tracked_object_labels_0.json | 40 ++- .../tracked_object_labels_1.json | 40 ++- .../tracked_object_labels_2.json | 40 ++- .../1/poses/city_SE3_egovehicle_0.json | 14 +- .../1/poses/city_SE3_egovehicle_1.json | 14 +- .../1/poses/city_SE3_egovehicle_2.json | 14 +- .../00000000-0000-0000-0000-000000000000.json | 65 +++- .../00000000-0000-0000-0000-000000000001.json | 65 +++- .../detection/1/vehicle_calibration_info.json | 305 +++++++++++++++++- .../tracked_object_labels_0.json | 42 ++- .../tracked_object_labels_1.json | 42 ++- .../tracked_object_labels_2.json | 42 ++- 13 files changed, 708 insertions(+), 19 deletions(-) diff --git a/tests/test_data/detection/1/city_info.json b/tests/test_data/detection/1/city_info.json index ec634ac4..9b447844 100644 --- a/tests/test_data/detection/1/city_info.json +++ b/tests/test_data/detection/1/city_info.json @@ -1 +1,3 @@ -{"city_name": "PIT"} +{ + "city_name": "PIT" +} diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json index 39f3c80a..af5a08fa 100644 --- a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -1,4 +1,40 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE"}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE"} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json index 11f63d81..1946f2c6 100644 --- a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -1,4 +1,40 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE"}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE"} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] diff --git a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json index 9b27b559..b7f7d52f 100644 --- a/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json +++ b/tests/test_data/detection/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -1,4 +1,40 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE"}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE"} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json index c8562dc8..41d5e3fb 100644 --- a/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_0.json @@ -1 +1,13 @@ -{"rotation": [0, 1, 0, 0], "translation": [1, 0, 0]} +{ + "rotation": [ + 0, + 1, + 0, + 0 + ], + "translation": [ + 1, + 0, + 0 + ] +} diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json index 07a39746..1c9920ec 100644 --- a/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_1.json @@ -1 +1,13 @@ -{"rotation": [1, 0, 0, 0], "translation": [2, 0, 0]} +{ + "rotation": [ + 1, + 0, + 0, + 0 + ], + "translation": [ + 2, + 0, + 0 + ] +} diff --git a/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json b/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json index c17eb32b..66e276e6 100644 --- a/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json +++ b/tests/test_data/detection/1/poses/city_SE3_egovehicle_2.json @@ -1 +1,13 @@ -{"rotation": [0, 0, 1, 0], "translation": [2, 1, 0]} +{ + "rotation": [ + 0, + 0, + 1, + 0 + ], + "translation": [ + 2, + 1, + 0 + ] +} diff --git a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json index 4fbfa530..3e64adb2 100644 --- a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json +++ b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000000.json @@ -1 +1,64 @@ -{"label_class": "VEHICLE", "uuid": "00000000-0000-0000-0000-000000000000", "log_id": "1", "track_label_frames": [{"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE"}, {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE"}, {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE"}]} +{ + "label_class": "VEHICLE", + "log_id": "1", + "track_label_frames": [ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2.0, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2.0, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + } + ], + "uuid": "00000000-0000-0000-0000-000000000000" +} diff --git a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json index 66fec672..d5e978c9 100644 --- a/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json +++ b/tests/test_data/detection/1/track_labels_amodal/00000000-0000-0000-0000-000000000001.json @@ -1 +1,64 @@ -{"label_class": "VEHICLE", "uuid": "00000000-0000-0000-0000-000000000001", "log_id": "1", "track_label_frames": [{"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE"}, {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE"}, {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2.0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE"}]} +{ + "label_class": "VEHICLE", + "log_id": "1", + "track_label_frames": [ + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2.0, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2.0, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } + ], + "uuid": "00000000-0000-0000-0000-000000000001" +} diff --git a/tests/test_data/detection/1/vehicle_calibration_info.json b/tests/test_data/detection/1/vehicle_calibration_info.json index 83c8a128..b9082574 100644 --- a/tests/test_data/detection/1/vehicle_calibration_info.json +++ b/tests/test_data/detection/1/vehicle_calibration_info.json @@ -1 +1,304 @@ -{"camera_data_": [{"key": "image_raw_stereo_front_right", "value": {"focal_length_x_px_": 3634.1664215146284, "focal_length_y_px_": 3634.1664215146284, "focal_center_x_px_": 1260.6433779229542, "focal_center_y_px_": 1059.5205903709868, "skew_": 0.0, "distortion_coefficients_": [-0.08785433857565009, -0.2963843841153259, 2.3674681480992428], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.5007328527621485, -0.495680186520672, 0.5013368080254209, -0.5022242206170416]}, "translation": [1.6421058178669956, -0.1499718687167596, 1.361891429230726]}}}, {"key": "image_raw_ring_rear_left", "value": {"focal_length_x_px_": 1393.318962096269, "focal_length_y_px_": 1393.318962096269, "focal_center_x_px_": 967.2847108481072, "focal_center_y_px_": 613.2332689161747, "skew_": 0.0, "distortion_coefficients_": [-0.17241307888830998, 0.11907008773383419, -0.027945198361512744], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.608056280997397, -0.6173801180944725, -0.3514622121439918, 0.35437785251920234]}, "translation": [1.105844707913899, 0.24650725363322967, 1.3735906821517418]}}}, {"key": "image_raw_stereo_front_left", "value": {"focal_length_x_px_": 3660.425411174431, "focal_length_y_px_": 3660.425411174431, "focal_center_x_px_": 1246.9202677240673, "focal_center_y_px_": 1063.4338217897928, "skew_": 0.0, "distortion_coefficients_": [-0.09737880168798231, 0.025226817408578506, 0.6478859260574071], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.49725711847079235, -0.5033139293759432, 0.4991231316939582, -0.5002864639726806]}, "translation": [1.619756836560706, 0.14842013734897835, 1.3648467699127986]}}}, {"key": "image_raw_ring_rear_right", "value": {"focal_length_x_px_": 1393.4158534327278, "focal_length_y_px_": 1393.4158534327278, "focal_center_x_px_": 951.6911354519806, "focal_center_y_px_": 597.3719168271283, "skew_": 0.0, "distortion_coefficients_": [-0.171660451822291, 0.11750204460346327, -0.027417995114452046], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.35852818774144446, -0.3519145089444871, -0.6125134020323175, 0.6102794845970104]}, "translation": [1.0985731208497924, -0.24534171331957055, 1.37160293757265]}}}, {"key": "image_raw_ring_side_left", "value": {"focal_length_x_px_": 1395.0857466752807, "focal_length_y_px_": 1395.0857466752807, "focal_center_x_px_": 967.7390137760952, "focal_center_y_px_": 613.3380881206222, "skew_": 0.0, "distortion_coefficients_": [-0.17228684997200724, 0.11667402061297209, -0.02528530850263856], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.7004567328555497, -0.7088187475676434, -0.05809163085288814, 0.05968007137075483]}, "translation": [1.2934806248356339, 0.27738067568824815, 1.371113292276382]}}}, {"key": "image_raw_ring_side_right", "value": {"focal_length_x_px_": 1391.654458212649, "focal_length_y_px_": 1391.654458212649, "focal_center_x_px_": 981.9614016400475, "focal_center_y_px_": 608.2719144486357, "skew_": 0.0, "distortion_coefficients_": [-0.1723517502956347, 0.11731566692457349, -0.02574725269922101], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.06401399257908719, -0.06266155729362148, -0.7078861012523953, 0.7006232979606847]}, "translation": [1.294530313917792, -0.28519924870913804, 1.3701008006525792]}}}, {"key": "image_raw_ring_front_right", "value": {"focal_length_x_px_": 1392.4445795594495, "focal_length_y_px_": 1392.4445795594495, "focal_center_x_px_": 958.0278877584801, "focal_center_y_px_": 608.506324610186, "skew_": 0.0, "distortion_coefficients_": [-0.1723521614846795, 0.12188989312717174, -0.03271981337763657], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.2653173446516557, -0.2685570788918094, 0.6608505371335351, -0.6486604424307151]}, "translation": [1.507413796878202, -0.2667563199332462, 1.3615052881313203]}}}, {"key": "image_raw_ring_front_center", "value": {"focal_length_x_px_": 1392.1069298937407, "focal_length_y_px_": 1392.1069298937407, "focal_center_x_px_": 980.1759848618066, "focal_center_y_px_": 604.3534182680304, "skew_": 0.0, "distortion_coefficients_": [-0.1720396447593493, 0.11689572230654095, -0.02511932396889168], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.49605542988442836, -0.49896196582115804, 0.5027901707576079, -0.5021633313331392]}, "translation": [1.6519358245144808, -0.0005354981581146487, 1.3613890006792675]}}}, {"key": "image_raw_ring_front_left", "value": {"focal_length_x_px_": 1393.7674787830663, "focal_length_y_px_": 1393.7674787830663, "focal_center_x_px_": 960.5401682856536, "focal_center_y_px_": 607.4003449968101, "skew_": 0.0, "distortion_coefficients_": [-0.17299777690816825, 0.1226893513068073, -0.03042136313516928], "vehicle_SE3_camera_": {"rotation": {"coefficients": [0.6480281666752452, -0.6560752002901659, 0.2739859090547071, -0.27305045028200403]}, "translation": [1.511797877348265, 0.25337046635825855, 1.3651560815686514]}}}], "vehicle_SE3_up_lidar_": {"rotation": {"coefficients": [0.9998848409086823, 0.0011389828365580806, -0.0010769852622658116, -0.015094626257773826]}, "translation": [1.3504724404154866, -0.003966310091761113, 1.5700506059693788]}, "vehicle_SE3_down_lidar_": {"rotation": {"coefficients": [5.6898203165648547e-05, -0.994250135333888, -0.10708208174631563, -0.0003048278454676608]}, "translation": [1.3502875642048289, -0.0019957763317242708, 1.4771699630675978]}} +{ + "camera_data_": [ + { + "key": "image_raw_stereo_front_right", + "value": { + "distortion_coefficients_": [ + -0.08785433857565009, + -0.2963843841153259, + 2.3674681480992428 + ], + "focal_center_x_px_": 1260.6433779229542, + "focal_center_y_px_": 1059.5205903709868, + "focal_length_x_px_": 3634.1664215146284, + "focal_length_y_px_": 3634.1664215146284, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.5007328527621485, + -0.495680186520672, + 0.5013368080254209, + -0.5022242206170416 + ] + }, + "translation": [ + 1.6421058178669956, + -0.1499718687167596, + 1.361891429230726 + ] + } + } + }, + { + "key": "image_raw_ring_rear_left", + "value": { + "distortion_coefficients_": [ + -0.17241307888830998, + 0.11907008773383419, + -0.027945198361512744 + ], + "focal_center_x_px_": 967.2847108481072, + "focal_center_y_px_": 613.2332689161747, + "focal_length_x_px_": 1393.318962096269, + "focal_length_y_px_": 1393.318962096269, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.608056280997397, + -0.6173801180944725, + -0.3514622121439918, + 0.35437785251920234 + ] + }, + "translation": [ + 1.105844707913899, + 0.24650725363322967, + 1.3735906821517418 + ] + } + } + }, + { + "key": "image_raw_stereo_front_left", + "value": { + "distortion_coefficients_": [ + -0.09737880168798231, + 0.025226817408578506, + 0.6478859260574071 + ], + "focal_center_x_px_": 1246.9202677240673, + "focal_center_y_px_": 1063.4338217897928, + "focal_length_x_px_": 3660.425411174431, + "focal_length_y_px_": 3660.425411174431, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.49725711847079235, + -0.5033139293759432, + 0.4991231316939582, + -0.5002864639726806 + ] + }, + "translation": [ + 1.619756836560706, + 0.14842013734897835, + 1.3648467699127986 + ] + } + } + }, + { + "key": "image_raw_ring_rear_right", + "value": { + "distortion_coefficients_": [ + -0.171660451822291, + 0.11750204460346327, + -0.027417995114452046 + ], + "focal_center_x_px_": 951.6911354519806, + "focal_center_y_px_": 597.3719168271283, + "focal_length_x_px_": 1393.4158534327278, + "focal_length_y_px_": 1393.4158534327278, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.35852818774144446, + -0.3519145089444871, + -0.6125134020323175, + 0.6102794845970104 + ] + }, + "translation": [ + 1.0985731208497924, + -0.24534171331957055, + 1.37160293757265 + ] + } + } + }, + { + "key": "image_raw_ring_side_left", + "value": { + "distortion_coefficients_": [ + -0.17228684997200724, + 0.11667402061297209, + -0.02528530850263856 + ], + "focal_center_x_px_": 967.7390137760952, + "focal_center_y_px_": 613.3380881206222, + "focal_length_x_px_": 1395.0857466752807, + "focal_length_y_px_": 1395.0857466752807, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.7004567328555497, + -0.7088187475676434, + -0.05809163085288814, + 0.05968007137075483 + ] + }, + "translation": [ + 1.2934806248356339, + 0.27738067568824815, + 1.371113292276382 + ] + } + } + }, + { + "key": "image_raw_ring_side_right", + "value": { + "distortion_coefficients_": [ + -0.1723517502956347, + 0.11731566692457349, + -0.02574725269922101 + ], + "focal_center_x_px_": 981.9614016400475, + "focal_center_y_px_": 608.2719144486357, + "focal_length_x_px_": 1391.654458212649, + "focal_length_y_px_": 1391.654458212649, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.06401399257908719, + -0.06266155729362148, + -0.7078861012523953, + 0.7006232979606847 + ] + }, + "translation": [ + 1.294530313917792, + -0.28519924870913804, + 1.3701008006525792 + ] + } + } + }, + { + "key": "image_raw_ring_front_right", + "value": { + "distortion_coefficients_": [ + -0.1723521614846795, + 0.12188989312717174, + -0.03271981337763657 + ], + "focal_center_x_px_": 958.0278877584801, + "focal_center_y_px_": 608.506324610186, + "focal_length_x_px_": 1392.4445795594495, + "focal_length_y_px_": 1392.4445795594495, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.2653173446516557, + -0.2685570788918094, + 0.6608505371335351, + -0.6486604424307151 + ] + }, + "translation": [ + 1.507413796878202, + -0.2667563199332462, + 1.3615052881313203 + ] + } + } + }, + { + "key": "image_raw_ring_front_center", + "value": { + "distortion_coefficients_": [ + -0.1720396447593493, + 0.11689572230654095, + -0.02511932396889168 + ], + "focal_center_x_px_": 980.1759848618066, + "focal_center_y_px_": 604.3534182680304, + "focal_length_x_px_": 1392.1069298937407, + "focal_length_y_px_": 1392.1069298937407, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.49605542988442836, + -0.49896196582115804, + 0.5027901707576079, + -0.5021633313331392 + ] + }, + "translation": [ + 1.6519358245144808, + -0.0005354981581146487, + 1.3613890006792675 + ] + } + } + }, + { + "key": "image_raw_ring_front_left", + "value": { + "distortion_coefficients_": [ + -0.17299777690816825, + 0.1226893513068073, + -0.03042136313516928 + ], + "focal_center_x_px_": 960.5401682856536, + "focal_center_y_px_": 607.4003449968101, + "focal_length_x_px_": 1393.7674787830663, + "focal_length_y_px_": 1393.7674787830663, + "skew_": 0.0, + "vehicle_SE3_camera_": { + "rotation": { + "coefficients": [ + 0.6480281666752452, + -0.6560752002901659, + 0.2739859090547071, + -0.27305045028200403 + ] + }, + "translation": [ + 1.511797877348265, + 0.25337046635825855, + 1.3651560815686514 + ] + } + } + } + ], + "vehicle_SE3_down_lidar_": { + "rotation": { + "coefficients": [ + 5.6898203165648547e-05, + -0.994250135333888, + -0.10708208174631563, + -0.0003048278454676608 + ] + }, + "translation": [ + 1.3502875642048289, + -0.0019957763317242708, + 1.4771699630675978 + ] + }, + "vehicle_SE3_up_lidar_": { + "rotation": { + "coefficients": [ + 0.9998848409086823, + 0.0011389828365580806, + -0.0010769852622658116, + -0.015094626257773826 + ] + }, + "translation": [ + 1.3504724404154866, + -0.003966310091761113, + 1.5700506059693788 + ] + } +} diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json index 28af00f7..5a63c6f7 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -1,4 +1,42 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 0, "label_class": "VEHICLE", "score": 1.0}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 0, "label_class": "VEHICLE", "score": 1.0} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json index b15b4a1f..63a5c633 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -1,4 +1,42 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 1, "label_class": "VEHICLE", "score": 1.0}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 0.0, "z": 1, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 1, "label_class": "VEHICLE", "score": 1.0} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json index 6bcb8d22..b5f11f76 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -1,4 +1,42 @@ [ - {"center": {"x": 0, "y": 0, "z": 0}, "rotation": {"x": 1.0, "y": 0.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000000", "timestamp": 2, "label_class": "VEHICLE", "score": 1.0}, - {"center": {"x": 2, "y": 2, "z": 0}, "rotation": {"x": 0.0, "y": 1.0, "z": 0, "w": 0}, "length": 2, "width": 2, "height": 2, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "timestamp": 2, "label_class": "VEHICLE", "score": 1.0} + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } ] From 89dba090e158d8e68b3a8073c75d44b62282d1e5 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 23 Sep 2020 10:26:44 -0400 Subject: [PATCH 033/113] Added iou function + added info to unit test docstrings. --- argoverse/evaluation/detection_utils.py | 38 ++++++++++----- argoverse/evaluation/eval_detection.py | 2 + tests/test_eval_detection.py | 63 ++++++++++++++++--------- 3 files changed, 71 insertions(+), 32 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 968851fe..9080395d 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -47,7 +47,7 @@ def filter_instances( """Filter the GT annotations based on a set of conditions (class name and distance from egovehicle). Args: - instances: The instances to be filtered (N,). + instances: The instances to be filtered (N, ). target_class_name: The name of the class of interest. filter_metric: The range metric used for filtering. max_detection_range: The maximum distance for range filtering. @@ -76,8 +76,8 @@ def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: dts: Detections (N,). Returns: - ranks: The ranking for the detections (N,). - scores: The detection scores (N,). + ranks: The ranking for the detections (N, ). + scores: The detection scores (N, ). """ scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] @@ -89,7 +89,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: """Interpolate the precision over all recall levels. Args: - prec: Precision at all recall levels (N,). + prec: Precision at all recall levels (N, ). method: Accumulation method. Returns: @@ -109,8 +109,8 @@ def compute_affinity_matrix( using a specified similarity function. Args: - dts: Detections (N,). - gts: Ground truth labels (M,). + dts: Detections (N, ). + gts: Ground truth labels (M, ). metric: Similarity metric type. Returns: @@ -129,8 +129,8 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar """Distance functions between detections and ground truth. Args: - dts: Detections (N,). - gts: Ground truth labels (M,). + dts: Detections (N, ). + gts: Ground truth labels (M, ). metric: Distance function type. Returns: @@ -144,9 +144,7 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar elif metric == DistFnType.SCALE: dt_dims = dts[["width", "length", "height"]] gt_dims = gts[["width", "length", "height"]] - inter = np.minimum(dt_dims, gt_dims).prod(axis=1) - union = np.maximum(dt_dims, gt_dims).prod(axis=1) - scale_errors = 1 - (inter / union) + scale_errors = 1 - iou_aligned_3d(dt_dims, gt_dims) return scale_errors elif metric == DistFnType.ORIENTATION: # re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw @@ -163,6 +161,24 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar raise NotImplemented("This distance metric is not implemented!") +def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: + """Calculate the 3d, axis-aligned (vertical axis alignment) intersection-over-union (IoU) + between the detections and the ground truth labels after aligning their poses. + + Args: + dt_dims: Detections (N, 3). + gt_dims: Ground truth labels (N, 3). + + Returns: + Intersection-over-union between the detections and their assigned ground + truth labels (N, ). + + """ + inter = np.minimum(dt_dims, gt_dims).prod(axis=1) + union = np.maximum(dt_dims, gt_dims).prod(axis=1) + return (inter / union).values + + def normalize_angle(angle: np.ndarray) -> np.ndarray: """Map angle (in radians) from domain [-π, π] to [0, π). diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 75b0b9a4..5b91d6bc 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -52,6 +52,8 @@ in addition to the mean statistics average across all classes, and P refers to the number of included statistics, e.g. AP, ATE, ASE, AOE, CDS by default. + Note: The `evaluate` function will use all available logical cores on the machine. + """ import argparse import logging diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 4a798f29..d3c09511 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -10,7 +10,7 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_affinity_matrix, dist_fn +from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_affinity_matrix, dist_fn, iou_aligned_3d from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator from argoverse.utils.transform import quat_scipy2argo_vectorized @@ -34,41 +34,62 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: def test_center_similarity() -> None: """Test that the Center similarity function works.""" - olr1 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) - olr2 = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) - assert compute_affinity_matrix(olr1, olr2, SimFnType.CENTER) == -5 + dts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) + gts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) + assert compute_affinity_matrix(dts, gts, SimFnType.CENTER) == -5 def test_translation_distance() -> None: - """Test that the translation distance function works.""" - df1 = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) - df2 = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) - assert dist_fn(df1, df2, DistFnType.TRANSLATION) == 75 ** (1 / 2) + """Intialize a detection and a ground truth label with only translation + parameters. Verify that calculated distance matches expected distance under + the specified `DistFnType`. + """ + dts = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) + gts = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) + assert dist_fn(dts, gts, DistFnType.TRANSLATION) == 75 ** (1 / 2) def test_scale_distance() -> None: - """Test that the scale distance function works.""" - df1 = DataFrame([{"width": 5, "height": 5, "length": 5}]) - df2 = DataFrame([{"width": 10, "height": 10, "length": 10}]) - assert (dist_fn(df1, df2, DistFnType.SCALE) == 1 - 0.125).all() + """Intialize a detection and a ground truth label with only shape + parameters (only shape parameters due to alignment assumption). + Verify that calculated scale error matches the expected value. + """ + dts = DataFrame([{"width": 5, "height": 5, "length": 5}]) + gts = DataFrame([{"width": 10, "height": 10, "length": 10}]) + assert (dist_fn(dts, gts, DistFnType.SCALE) == 1 - 0.125).all() def test_orientation_distance() -> None: - """Test that the orientation distance function works.""" + """Intialize a detection and a ground truth label with only orientation + parameters. Verify that calculated orientation error matches the expected + smallest angle between the detection and ground truth label. + """ # check all of the 45 degree angles vecs_45_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 4)] for i in range(len(vecs_45_apart) - 1): - df1 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) - df2 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) - assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 4) - assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 4) + dts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) + gts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) + assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), np.pi / 4) + assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), np.pi / 4) # check all of the 90 degree angles vecs_90_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 2)] for i in range(len(vecs_90_apart) - 1): - df1 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) - df2 = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) - assert np.isclose(dist_fn(df1, df2, DistFnType.ORIENTATION), np.pi / 2) - assert np.isclose(dist_fn(df2, df1, DistFnType.ORIENTATION), np.pi / 2) + dts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) + gts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) + assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), np.pi / 2) + assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), np.pi / 2) + + +def test_iou_aligned_3d() -> None: + """Intialize a detection and a ground truth label with only shape + parameters (only shape parameters due to alignment assumption). + Verify that calculated intersection-over-union matches the expected + value between the detection and ground truth label. + """ + dt_dims = DataFrame([{"width": 10, "height": 3, "length": 4}]) + gt_dims = DataFrame([{"width": 5, "height": 2, "length": 9}]) + + assert (iou_aligned_3d(dt_dims, gt_dims) == (40.0 / 270.0)).all() def test_ap(metrics: DataFrame) -> None: From 52d826975dd5dd1e6de6070ebe98fa94c2233c47 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 23 Sep 2020 10:29:21 -0400 Subject: [PATCH 034/113] Fixed naming + missed docstring. --- argoverse/evaluation/detection_utils.py | 8 ++++---- argoverse/evaluation/eval_detection.py | 4 ++-- tests/test_eval_detection.py | 8 +++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 9080395d..e06462e2 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -23,7 +23,7 @@ from argoverse.utils.transform import quat_argo2scipy_vectorized -class SimFnType(Enum): +class AffFnType(Enum): CENTER = auto() @@ -103,10 +103,10 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: def compute_affinity_matrix( - dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: SimFnType + dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: AffFnType ) -> np.ndarray: """Calculate the match matrix between detections and ground truth labels, - using a specified similarity function. + using a specified affinity function. Args: dts: Detections (N, ). @@ -116,7 +116,7 @@ def compute_affinity_matrix( Returns: sims: Similarity scores between detections and ground truth annotations (N, M). """ - if metric == SimFnType.CENTER: + if metric == AffFnType.CENTER: dt_centers = np.array([dt.translation for dt in dts]) gt_centers = np.array([gt.translation for gt in gts]) sims = -cdist(dt_centers, gt_centers) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 5b91d6bc..91baae0e 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -72,9 +72,9 @@ from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT from argoverse.data_loading.object_label_record import read_label from argoverse.evaluation.detection_utils import ( + AffFnType, DistFnType, FilterMetric, - SimFnType, compute_affinity_matrix, dist_fn, filter_instances, @@ -125,7 +125,7 @@ class DetectionCfg: """ affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) - affinity_fn_type: SimFnType = SimFnType.CENTER + affinity_fn_type: AffFnType = AffFnType.CENTER n_rec_samples: int = 101 tp_thresh: float = 2.0 significant_digits: int = 3 diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index d3c09511..155bc0e3 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -10,7 +10,7 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.evaluation.detection_utils import DistFnType, SimFnType, compute_affinity_matrix, dist_fn, iou_aligned_3d +from argoverse.evaluation.detection_utils import AffFnType, DistFnType, compute_affinity_matrix, dist_fn, iou_aligned_3d from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator from argoverse.utils.transform import quat_scipy2argo_vectorized @@ -33,10 +33,12 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: def test_center_similarity() -> None: - """Test that the Center similarity function works.""" + """Intialize a detection and a ground truth label. Verify that calculated distance matches expected affinity + under the specified `AffFnType`. + """ dts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) gts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) - assert compute_affinity_matrix(dts, gts, SimFnType.CENTER) == -5 + assert compute_affinity_matrix(dts, gts, AffFnType.CENTER) == -5 def test_translation_distance() -> None: From 06534468b85179b56b64dae9fc240674e3073e41 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 23 Sep 2020 10:38:55 -0400 Subject: [PATCH 035/113] Added additional comments/clarifications. --- argoverse/evaluation/eval_detection.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 91baae0e..f308c551 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -103,6 +103,8 @@ MAX_NORM_ASE: float = 1.0 MAX_NORM_AOE: float = 1.0 MIN_CDS: float = 0.0 + +# Each measure is in [0, 1]. MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORM_ATE, MAX_NORM_ASE, MAX_NORM_AOE, MIN_CDS] MAX_NUM_BOXES: int = 500 @@ -120,18 +122,19 @@ class DetectionCfg: significant_digits: The precision for metrics. detection_classes: Detection classes for evaluation. detection_metric: The detection metric to use for filtering of both detections and ground truth annotations. - max_detection_range: The max distance (under a specific metric) for a a detection or ground truth to be considered for evaluation. + max_detection_range: The max distance (under a specific metric in meters) for a a detection or ground truth to be + considered for evaluation. save_figs: Flag to save figures. """ - affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) + affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) # Meters affinity_fn_type: AffFnType = AffFnType.CENTER n_rec_samples: int = 101 - tp_thresh: float = 2.0 + tp_thresh: float = 2.0 # Meters significant_digits: int = 3 detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) detection_metric: FilterMetric = FilterMetric.EUCLIDEAN - max_detection_range: float = 100.0 + max_detection_range: float = 100.0 # Meters save_figs: bool = False tp_normalization_terms: np.ndarray = field(init=False) From 69939064485216f5ef7e4edf6e3a2dcc09cda87e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 17:46:43 +0000 Subject: [PATCH 036/113] Added comments, updated variable names. --- argoverse/evaluation/detection_utils.py | 3 ++- argoverse/evaluation/eval_detection.py | 12 ++++++------ tests/test_eval_detection.py | 10 ++++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index e06462e2..ac96aa39 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -163,7 +163,8 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: """Calculate the 3d, axis-aligned (vertical axis alignment) intersection-over-union (IoU) - between the detections and the ground truth labels after aligning their poses. + between the detections and the ground truth labels. Both objects are aligned to their + +x axis and their centroids are placed at the origin before computation of the IoU. Args: dt_dims: Detections (N, 3). diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index f308c551..0c13e655 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -96,16 +96,16 @@ STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] -MAX_YAW_ERROR = np.pi +MAX_YAW_ERROR: float = np.pi MIN_AP: float = 0.0 -MAX_NORM_ATE: float = 1.0 -MAX_NORM_ASE: float = 1.0 -MAX_NORM_AOE: float = 1.0 +MAX_NORMALIZED_ATE: float = 1.0 +MAX_NORMALIZED_ASE: float = 1.0 +MAX_NORMALIZED_AOE: float = 1.0 MIN_CDS: float = 0.0 # Each measure is in [0, 1]. -MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORM_ATE, MAX_NORM_ASE, MAX_NORM_AOE, MIN_CDS] +MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORMALIZED_ATE, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS] MAX_NUM_BOXES: int = 500 @@ -122,7 +122,7 @@ class DetectionCfg: significant_digits: The precision for metrics. detection_classes: Detection classes for evaluation. detection_metric: The detection metric to use for filtering of both detections and ground truth annotations. - max_detection_range: The max distance (under a specific metric in meters) for a a detection or ground truth to be + max_detection_range: The max distance (under a specific metric in meters) for a detection or ground truth to be considered for evaluation. save_figs: Flag to save figures. """ diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 155bc0e3..12c576b9 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -32,12 +32,12 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: return evaluator.evaluate() -def test_center_similarity() -> None: +def test_affinity_center() -> None: """Intialize a detection and a ground truth label. Verify that calculated distance matches expected affinity under the specified `AffFnType`. """ - dts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)]) - gts = np.array([ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)]) + dts = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)] + gts = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)] assert compute_affinity_matrix(dts, gts, AffFnType.CENTER) == -5 @@ -91,7 +91,9 @@ def test_iou_aligned_3d() -> None: dt_dims = DataFrame([{"width": 10, "height": 3, "length": 4}]) gt_dims = DataFrame([{"width": 5, "height": 2, "length": 9}]) - assert (iou_aligned_3d(dt_dims, gt_dims) == (40.0 / 270.0)).all() + # Intersection is 40 = 4 * 5 * 2 (min of all dimensions). + # Union is the sum of the two volumes, minus intersection: 270 = (10 * 3 * 4) + (5 * 2 * 9) - 40. + assert (iou_aligned_3d(dt_dims, gt_dims) == (40 / 270.0)).all() def test_ap(metrics: DataFrame) -> None: From a2863eafd037ebcb8f179af4139ceac900937714 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 17:49:48 +0000 Subject: [PATCH 037/113] Added constant comments. --- argoverse/evaluation/eval_detection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 0c13e655..2c4f6fb1 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -98,11 +98,14 @@ MAX_YAW_ERROR: float = np.pi +# Higher is better MIN_AP: float = 0.0 +MIN_CDS: float = 0.0 + +# Lower is better MAX_NORMALIZED_ATE: float = 1.0 MAX_NORMALIZED_ASE: float = 1.0 MAX_NORMALIZED_AOE: float = 1.0 -MIN_CDS: float = 0.0 # Each measure is in [0, 1]. MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORMALIZED_ATE, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS] From ecb37f55630a9459f48b02215888a795cece9bdc Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 18:28:01 +0000 Subject: [PATCH 038/113] Naming fixes, docstring fixes, minor type cleanup. --- argoverse/evaluation/eval_detection.py | 18 +++--- tests/test_eval_detection.py | 87 +++++++++++++++++--------- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 2c4f6fb1..4eee8a17 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -98,11 +98,11 @@ MAX_YAW_ERROR: float = np.pi -# Higher is better +# Higher is better. MIN_AP: float = 0.0 MIN_CDS: float = 0.0 -# Lower is better +# Lower is better. MAX_NORMALIZED_ATE: float = 1.0 MAX_NORMALIZED_ASE: float = 1.0 MAX_NORMALIZED_AOE: float = 1.0 @@ -110,8 +110,11 @@ # Each measure is in [0, 1]. MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORMALIZED_ATE, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS] +# Max number of boxes considered per class per scene. MAX_NUM_BOXES: int = 500 +SIGNIFICANT_DIGITS: float = 3 + @dataclass class DetectionCfg: @@ -122,19 +125,18 @@ class DetectionCfg: affinity_fn_type: The type of affinity function to be used for calculating average precision. n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. tp_thresh: Center distance threshold for the true positive metrics (in meters). - significant_digits: The precision for metrics. detection_classes: Detection classes for evaluation. detection_metric: The detection metric to use for filtering of both detections and ground truth annotations. max_detection_range: The max distance (under a specific metric in meters) for a detection or ground truth to be considered for evaluation. save_figs: Flag to save figures. + tp_normalization_terms: Normalization constants for ATE, ASE, and AOE. """ affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) # Meters affinity_fn_type: AffFnType = AffFnType.CENTER n_rec_samples: int = 101 tp_thresh: float = 2.0 # Meters - significant_digits: int = 3 detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) detection_metric: FilterMetric = FilterMetric.EUCLIDEAN max_detection_range: float = 100.0 # Meters @@ -195,10 +197,10 @@ def evaluate(self) -> pd.DataFrame: ) summary.update(summary_update) - summary = summary.round(self.detection_cfg.significant_digits) + summary = summary.round(SIGNIFICANT_DIGITS) summary.index = summary.index.str.title() - summary.loc["Average Metrics"] = summary.mean().round(self.detection_cfg.significant_digits) + summary.loc["Average Metrics"] = summary.mean().round(SIGNIFICANT_DIGITS) return summary def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: @@ -275,7 +277,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) - # Get the most similar GT label for each detection. + # Get the GT label for each max-affinity GT label, detection pair. gt_matches = affinity_matrix.argmax(axis=1)[np.newaxis, :] # The affinity matrix is an N by M matrix of the detections and ground truth labels respectively. @@ -287,7 +289,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) for i, thr in enumerate(self.detection_cfg.affinity_threshs): - # tp_mask may need to be defined differently with other similarity metrics + # tp_mask may need to be defined differently with other affinities tp_mask = affinities[unique_dt_matches] > -thr metrics[unique_dt_matches, i] = tp_mask diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 12c576b9..f7c7b3f9 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -3,6 +3,7 @@ import logging import pathlib +from typing import List import numpy as np import pytest @@ -11,7 +12,7 @@ from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.evaluation.detection_utils import AffFnType, DistFnType, compute_affinity_matrix, dist_fn, iou_aligned_3d -from argoverse.evaluation.eval_detection import DetectionCfg, DetectionEvaluator +from argoverse.evaluation.eval_detection import SIGNIFICANT_DIGITS, DetectionCfg, DetectionEvaluator from argoverse.utils.transform import quat_scipy2argo_vectorized TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection" @@ -21,7 +22,7 @@ @pytest.fixture def evaluator() -> DetectionEvaluator: """Define an evaluator that compares a set of results to itself.""" - detection_cfg = DetectionCfg(significant_digits=32, detection_classes=["VEHICLE"]) + detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) @@ -36,9 +37,11 @@ def test_affinity_center() -> None: """Intialize a detection and a ground truth label. Verify that calculated distance matches expected affinity under the specified `AffFnType`. """ - dts = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)] - gts = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)] - assert compute_affinity_matrix(dts, gts, AffFnType.CENTER) == -5 + dts: List[ObjectLabelRecord] = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)] + gts: List[ObjectLabelRecord] = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)] + + expected_result: float = -5 + assert compute_affinity_matrix(dts, gts, AffFnType.CENTER) == expected_result def test_translation_distance() -> None: @@ -46,9 +49,11 @@ def test_translation_distance() -> None: parameters. Verify that calculated distance matches expected distance under the specified `DistFnType`. """ - dts = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) - gts = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) - assert dist_fn(dts, gts, DistFnType.TRANSLATION) == 75 ** (1 / 2) + dts: DataFrame = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) + gts: DataFrame = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) + + expected_result: float = 75.0 ** (1 / 2) + assert dist_fn(dts, gts, DistFnType.TRANSLATION) == expected_result def test_scale_distance() -> None: @@ -56,30 +61,51 @@ def test_scale_distance() -> None: parameters (only shape parameters due to alignment assumption). Verify that calculated scale error matches the expected value. """ - dts = DataFrame([{"width": 5, "height": 5, "length": 5}]) - gts = DataFrame([{"width": 10, "height": 10, "length": 10}]) - assert (dist_fn(dts, gts, DistFnType.SCALE) == 1 - 0.125).all() + dts: DataFrame = DataFrame([{"width": 5, "height": 5, "length": 5}]) + gts: DataFrame = DataFrame([{"width": 10, "height": 10, "length": 10}]) + + expected_result: float = 1 - 0.125 + assert dist_fn(dts, gts, DistFnType.SCALE) == expected_result -def test_orientation_distance() -> None: +def test_orientation_quarter_angles() -> None: """Intialize a detection and a ground truth label with only orientation parameters. Verify that calculated orientation error matches the expected - smallest angle between the detection and ground truth label. + smallest angle ((2 * np.pi) / 4) between the detection and ground truth label. """ - # check all of the 45 degree angles - vecs_45_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 4)] - for i in range(len(vecs_45_apart) - 1): - dts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i]).as_quat())}]) - gts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_45_apart[i + 1]).as_quat())}]) - assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), np.pi / 4) - assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), np.pi / 4) - # check all of the 90 degree angles - vecs_90_apart = [angle * np.array([0, 0, 1]) for angle in np.arange(0, 2 * np.pi, np.pi / 2)] - for i in range(len(vecs_90_apart) - 1): - dts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i]).as_quat())}]) - gts = DataFrame([{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(vecs_90_apart[i + 1]).as_quat())}]) - assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), np.pi / 2) - assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), np.pi / 2) + + # Check all of the 90 degree angles + expected_result: float = (2 * np.pi) / 4 + quarter_angles = [np.array([0, 0, angle]) for angle in np.arange(0, 2 * np.pi, expected_result)] + for i in range(len(quarter_angles) - 1): + dts: DataFrame = DataFrame( + [{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(quarter_angles[i]).as_quat())}] + ) + gts: DataFrame = DataFrame( + [{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(quarter_angles[i + 1]).as_quat())}] + ) + + assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), expected_result) + assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), expected_result) + + +def test_orientation_eigth_angles() -> None: + """Intialize a detection and a ground truth label with only orientation + parameters. Verify that calculated orientation error matches the expected + smallest angle ((2 * np.pi) / 8) between the detection and ground truth label. + """ + expected_result: float = (2 * np.pi) / 8 + eigth_angle = [np.array([0, 0, angle]) for angle in np.arange(0, 2 * np.pi, expected_result)] + for i in range(len(eigth_angle) - 1): + dts: DataFrame = DataFrame( + [{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(eigth_angle[i]).as_quat())}] + ) + gts: DataFrame = DataFrame( + [{"quaternion": quat_scipy2argo_vectorized(R.from_rotvec(eigth_angle[i + 1]).as_quat())}] + ) + + assert np.isclose(dist_fn(dts, gts, DistFnType.ORIENTATION), expected_result) + assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), expected_result) def test_iou_aligned_3d() -> None: @@ -88,12 +114,13 @@ def test_iou_aligned_3d() -> None: Verify that calculated intersection-over-union matches the expected value between the detection and ground truth label. """ - dt_dims = DataFrame([{"width": 10, "height": 3, "length": 4}]) - gt_dims = DataFrame([{"width": 5, "height": 2, "length": 9}]) + dt_dims: DataFrame = DataFrame([{"width": 10, "height": 3, "length": 4}]) + gt_dims: DataFrame = DataFrame([{"width": 5, "height": 2, "length": 9}]) # Intersection is 40 = 4 * 5 * 2 (min of all dimensions). # Union is the sum of the two volumes, minus intersection: 270 = (10 * 3 * 4) + (5 * 2 * 9) - 40. - assert (iou_aligned_3d(dt_dims, gt_dims) == (40 / 270.0)).all() + expected_result: float = 40 / 270.0 + assert iou_aligned_3d(dt_dims, gt_dims) == expected_result def test_ap(metrics: DataFrame) -> None: From 9e39da80bb5d211829bc2a8ea9e0f41c11b0eae2 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 18:30:57 +0000 Subject: [PATCH 039/113] Updated to be consistent with other tests. --- tests/test_eval_detection.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index f7c7b3f9..ab091743 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -125,19 +125,23 @@ def test_iou_aligned_3d() -> None: def test_ap(metrics: DataFrame) -> None: """Test that AP is 1 for the self-compared results.""" - assert metrics.AP.loc["Average Metrics"] == 1 + expected_result: float = 1.0 + assert metrics.AP.loc["Average Metrics"] == expected_result def test_translation_error(metrics: DataFrame) -> None: """Test that ATE is 0 for the self-compared results.""" - assert metrics.ATE.loc["Average Metrics"] == 0 + expected_result: float = 0.0 + assert metrics.ATE.loc["Average Metrics"] == expected_result def test_scale_error(metrics: DataFrame) -> None: """Test that ASE is 0 for the self-compared results.""" - assert metrics.ASE.loc["Average Metrics"] == 0 + expected_result: float = 0.0 + assert metrics.ASE.loc["Average Metrics"] == expected_result def test_orientation_error(metrics: DataFrame) -> None: """Test that AOE is 0 for the self-compared results.""" - assert metrics.AOE.loc["Average Metrics"] == 0 + expected_result: float = 0.0 + assert metrics.AOE.loc["Average Metrics"] == expected_result From 632c9b7f1b9d86638c235fc11467cdac22a635fc Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 21:00:50 +0000 Subject: [PATCH 040/113] Shape fixes, test style fixes. --- argoverse/evaluation/detection_utils.py | 4 ++-- argoverse/evaluation/eval_detection.py | 2 -- tests/test_eval_detection.py | 24 +++++++++++++++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index ac96aa39..a1caa87b 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -130,11 +130,11 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar Args: dts: Detections (N, ). - gts: Ground truth labels (M, ). + gts: Ground truth labels (N, ). metric: Distance function type. Returns: - Distance between the detections and ground truth, using the provided metric. + Distance between the detections and ground truth, using the provided metric (N, ). """ if metric == DistFnType.TRANSLATION: dt_centers = np.vstack(dts["translation"].array) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 4eee8a17..c44f1a2c 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -17,10 +17,8 @@ 1. Average Translation Error: The average Euclidean distance (center-based) between a detection and its ground truth assignment. - 2. Average Scale Error: The average intersection over union (IoU) after the prediction and assigned ground truth's pose has been aligned. - 3. Average Orientation Error: The average angular distance between the detection and the assigned ground truth. We choose the smallest angle between the two different headings when calculating the error. diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index ab091743..31adba06 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -37,8 +37,26 @@ def test_affinity_center() -> None: """Intialize a detection and a ground truth label. Verify that calculated distance matches expected affinity under the specified `AffFnType`. """ - dts: List[ObjectLabelRecord] = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([0, 0, 0]), 5.0, 5.0, 5.0, 0)] - gts: List[ObjectLabelRecord] = [ObjectLabelRecord(np.array([0, 0, 0, 0]), np.array([3, 4, 0]), 5.0, 5.0, 5.0, 0)] + dts: List[ObjectLabelRecord] = [ + ObjectLabelRecord( + quaternion=np.array([0, 0, 0, 0]), + translation=np.array([0, 0, 0]), + length=5.0, + width=5.0, + height=5.0, + occlusion=0, + ) + ] + gts: List[ObjectLabelRecord] = [ + ObjectLabelRecord( + quaternion=np.array([0, 0, 0, 0]), + translation=np.array([3, 4, 0]), + length=5.0, + width=5.0, + height=5.0, + occlusion=0, + ) + ] expected_result: float = -5 assert compute_affinity_matrix(dts, gts, AffFnType.CENTER) == expected_result @@ -52,7 +70,7 @@ def test_translation_distance() -> None: dts: DataFrame = DataFrame([{"translation": [0.0, 0.0, 0.0]}]) gts: DataFrame = DataFrame([{"translation": [5.0, 5.0, 5.0]}]) - expected_result: float = 75.0 ** (1 / 2) + expected_result: float = np.sqrt(25 + 25 + 25) assert dist_fn(dts, gts, DistFnType.TRANSLATION) == expected_result From 1415d89cd3bd1e1b244068886eead9f20b305a34 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 23 Sep 2020 21:01:50 +0000 Subject: [PATCH 041/113] Docstring fix. --- argoverse/evaluation/detection_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index a1caa87b..275f878a 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -70,7 +70,7 @@ def filter_instances( def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: - """Get the rankings for the detections. + """Get the rankings for the detections, according to detector confidence. Args: dts: Detections (N,). From 93a0df0668c77cccfa091fad446475aa36ef51fe Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 24 Sep 2020 00:46:06 +0000 Subject: [PATCH 042/113] Fixed docstrings and renamed some variables. --- argoverse/evaluation/eval_detection.py | 33 ++++++++++++++++---------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index c44f1a2c..caf94aad 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -13,7 +13,8 @@ All true positive errors -- as the name implies -- accumulate error solely when an object is a true positive match to a ground truth detection. The matching - criterion is represented by `tp_thresh` in the DetectionCfg class. + criterion is represented by `tp_thresh` in the DetectionCfg class. In our challege, + we use a `tp_thresh` of 2 meters. 1. Average Translation Error: The average Euclidean distance (center-based) between a detection and its ground truth assignment. @@ -202,15 +203,16 @@ def evaluate(self) -> pd.DataFrame: return summary def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: - """Accumulate the statistics for each LiDAR frame. + """Accumulate the true/false positives (boolean flags) and true positive errors for each class. Args: gt_fpath: Ground truth file path. Returns: - cls_stats: Class statistics of shape ((N, K + S) where K is the number of true positive thresholds used - for AP computation and S is the number of true positive errors. - cls_to_ninst: Mapping of the class names to the number of instances in the ground + cls_to_accum: Class to accumulated statistics dictionary of shape |C| -> (N, K + S) where C + is the number of detection classes, K is the number of true positive thresholds used for + AP computation, and S is the number of true positive errors. + cls_to_ninst: Mapping of shape |C| -> (1, ) the class names to the number of instances in the ground truth dataset. """ log_id = gt_fpath.parents[1].stem @@ -222,7 +224,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa dts = np.array(read_label(str(dt_fpath))) gts = np.array(read_label(str(gt_fpath))) - cls_stats = defaultdict(list) + cls_to_accum = defaultdict(list) cls_to_ninst = defaultdict(int) for class_name in self.detection_cfg.detection_classes: dt_filtered = filter_instances( @@ -243,10 +245,10 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa if dt_filtered.shape[0] > 0: ranked_detections, scores = rank(dt_filtered) metrics = self.assign(ranked_detections, gt_filtered) - cls_stats[class_name] = np.hstack((metrics, scores)) + cls_to_accum[class_name] = np.hstack((metrics, scores)) cls_to_ninst[class_name] = gt_filtered.shape[0] - return cls_stats, cls_to_ninst + return cls_to_accum, cls_to_ninst def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: """Attempt assignment of each detection to a ground truth label. @@ -285,13 +287,18 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) - for i, thr in enumerate(self.detection_cfg.affinity_threshs): + for i, thresh in enumerate(self.detection_cfg.affinity_threshs): - # tp_mask may need to be defined differently with other affinities - tp_mask = affinities[unique_dt_matches] > -thr + # `tp_mask` may need to be defined differently with other affinities + tp_mask = affinities[unique_dt_matches] > -thresh metrics[unique_dt_matches, i] = tp_mask - if thr == self.detection_cfg.tp_thresh and np.count_nonzero(tp_mask) > 0: + # Only compute true positive error when `thresh` is equal to the tp threshold. + is_tp_thresh = thresh == self.detection_cfg.tp_thresh + # Ensure that there are true positives of the respective class in the frame. + has_true_positives = np.count_nonzero(tp_mask) > 0 + + if is_tp_thresh and has_true_positives: dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] @@ -393,7 +400,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("-d", "--dt_fpath", type=str, help="Detection root folder path.", required=True) parser.add_argument("-g", "--gt_fpath", type=str, help="Ground truth root folder path.", required=True) - parser.add_argument("-f", "--fig_fpath", type=str, help="Figures root folder path.", default="figs/") + parser.add_argument("-f", "--fig_fpath", type=str, help="Figures root folder path.", default="figs") args = parser.parse_args() logger.info(f"args == {args}") From efb1faa504cdee3b99b26cbfe83c405f90dd254f Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 24 Sep 2020 03:51:49 +0000 Subject: [PATCH 043/113] Implemented function. Retrofitted and to use vectorized code. --- argoverse/evaluation/detection_utils.py | 30 ++++++++----- argoverse/evaluation/eval_tracking.py | 53 +++++----------------- argoverse/utils/json_utils.py | 4 +- tests/test_eval_tracking.py | 59 +++++++++++++------------ 4 files changed, 62 insertions(+), 84 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 275f878a..96eeb152 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -19,7 +19,6 @@ from scipy.spatial.transform import Rotation as R from argoverse.data_loading.object_label_record import ObjectLabelRecord -from argoverse.evaluation.eval_tracking import get_orientation_error_deg from argoverse.utils.transform import quat_argo2scipy_vectorized @@ -147,15 +146,14 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar scale_errors = 1 - iou_aligned_3d(dt_dims, gt_dims) return scale_errors elif metric == DistFnType.ORIENTATION: - # re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw + # Re-order quaternions to go from Argoverse format to scipy format, then the third euler angle (z) is yaw. dt_quats = np.vstack(dts["quaternion"].array) dt_yaws = R.from_quat(quat_argo2scipy_vectorized(dt_quats)).as_euler("xyz")[:, 2] gt_quats = np.vstack(gts["quaternion"].array) gt_yaws = R.from_quat(quat_argo2scipy_vectorized(gt_quats)).as_euler("xyz")[:, 2] - signed_orientation_errors = normalize_angle(dt_yaws - gt_yaws) - orientation_errors = np.abs(signed_orientation_errors) + orientation_errors = wrap(dt_yaws - gt_yaws) return orientation_errors else: raise NotImplemented("This distance metric is not implemented!") @@ -180,12 +178,24 @@ def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: return (inter / union).values -def normalize_angle(angle: np.ndarray) -> np.ndarray: - """Map angle (in radians) from domain [-π, π] to [0, π). +def wrap(angles: np.ndarray, period: float = np.pi) -> np.ndarray: + """Map angles (in radians) from domain [-∞, ∞] to [0, π). This function is + the inverse of `np.unwrap`. Returns: - The angle (in radians) mapped to the interval [0, π]. + The angles (in radians) mapped to the interval [0, π). """ - period = 2 * np.pi - phase_shift = np.pi - return (angle + np.pi) % period - phase_shift + + # Map angles to [0, ∞]. + angles = np.abs(angles) + + # Calculate floor division and remainder simultaneously. + divs, mods = np.divmod(angles, period) + + # Select angles which exceed specified period. + angle_complement_mask = np.nonzero(divs) + + # Take set complement of `mods` w.r.t. the set [0, π]. + # `mods` must be nonzero, thus the image is the interval [0, π). + angles[angle_complement_mask] = period - mods[angle_complement_mask] + return angles diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index d288b68e..6a376e49 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -6,7 +6,7 @@ import pathlib import pickle from pathlib import Path -from typing import Any, Dict, List, TextIO, Union +from typing import Any, Dict, List, Optional, TextIO, Union import motmetrics as mm import numpy as np @@ -14,7 +14,9 @@ from argoverse.evaluation.eval_utils import label_to_bbox from argoverse.utils.json_utils import read_json_file -from argoverse.utils.se3 import SE3 +from click.core import Option + +from .detection_utils import wrap mh = mm.metrics.create() logger = logging.getLogger(__name__) @@ -78,45 +80,6 @@ def get_distance_iou_3d(x1: np.ndarray, x2: np.ndarray, name: str = "bbox") -> f return float(score) -def get_orientation_error_deg( - yaw1: Union[float, np.ndarray], yaw2: Union[float, np.ndarray] -) -> Union[float, np.ndarray]: - """ - Compute the smallest difference between 2 angles, in magnitude (absolute difference). - First, find the difference between the two yaw angles; since - each angle is guaranteed to be [-pi,pi] as the output of arctan2, then their - difference is bounded to [-2pi,2pi]. - - If the difference exceeds pi, then its corresponding angle in [-pi,0] - would be smaller in magnitude. On the other hand, if the difference is - less than -pi degrees, then we are guaranteed its counterpart in [0,pi] - would be smaller in magnitude. - - Ref: - https://stackoverflow.com/questions/1878907/the-smallest-difference-between-2-angles - - Args: - - yaw1: angle around unit circle, in radians in [-pi,pi] - - yaw2: angle around unit circle, in radians in [-pi,pi] - - Returns: - - error: smallest difference between 2 angles, in degrees - """ - EPSILON = 1e-5 - assert -(np.pi + EPSILON) < yaw1 and yaw1 < (np.pi + EPSILON) - assert -(np.pi + EPSILON) < yaw2 and yaw2 < (np.pi + EPSILON) - - error = np.rad2deg(yaw1 - yaw2) - if error > 180: - error -= 360 - if error < -180: - error += 360 - - # get positive angle difference instead of signed angle difference - error = np.abs(error) - return float(error) - - def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str) -> float: """Get the distance between two poses, returns nan if distance is larger than detection threshold. @@ -134,7 +97,11 @@ def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str elif name == "iou": return get_distance_iou_3d(x1, x2, name) elif name == "orientation": - return get_orientation_error_deg(x1["orientation"], x2["orientation"]) + theta = np.array([x1["orientation"]] - x2["orientation"]) + dist = wrap(theta).item() + + # Convert to degrees. + return np.rad2deg(dist) else: raise ValueError("Not implemented..") @@ -146,7 +113,7 @@ def eval_tracks( d_max: float, out_file: TextIO, centroid_method: str, - diffatt: str, + diffatt: Optional[str], category: str = "VEHICLE", ) -> None: """Evaluate tracking output. diff --git a/argoverse/utils/json_utils.py b/argoverse/utils/json_utils.py index 989b3f10..b25eced8 100644 --- a/argoverse/utils/json_utils.py +++ b/argoverse/utils/json_utils.py @@ -3,7 +3,7 @@ import json import os -from typing import Any, Dict, Union +from typing import Any, Dict, List, Union def read_json_file(fpath: Union[str, "os.PathLike[str]"]) -> Any: @@ -19,7 +19,7 @@ def read_json_file(fpath: Union[str, "os.PathLike[str]"]) -> Any: return json.load(f) -def save_json_dict(json_fpath: Union[str, "os.PathLike[str]"], dictionary: Dict[Any, Any]) -> None: +def save_json_dict(json_fpath: Union[str, "os.PathLike[str]"], dictionary: Union[Dict[Any, Any], List[Any]]) -> None: """Save a Python dictionary to a JSON file. Args: diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py index bdae6b8b..f04021d2 100644 --- a/tests/test_eval_tracking.py +++ b/tests/test_eval_tracking.py @@ -1,15 +1,16 @@ -#!/usr/bin/python3 +# import os import shutil -from collections import defaultdict, namedtuple +from collections import defaultdict from pathlib import Path -from typing import Any, Mapping, NamedTuple, Tuple +from typing import Any, DefaultDict, Dict, List, Mapping, NamedTuple, Tuple import numpy as np from scipy.spatial.transform import Rotation -from argoverse.evaluation.eval_tracking import eval_tracks, get_orientation_error_deg +from argoverse.evaluation.detection_utils import wrap +from argoverse.evaluation.eval_tracking import eval_tracks from argoverse.utils.json_utils import save_json_dict _ROOT = Path(__file__).resolve().parent @@ -72,7 +73,7 @@ class TrackedObjRec(NamedTuple): class TrackedObjects: def __init__(self, log_id: str, is_gt: bool) -> None: """ """ - self.ts_to_trackedlabels_dict = defaultdict(list) + self.ts_to_trackedlabels_dict: DefaultDict[int, List[Dict]] = defaultdict(list) self.log_id = log_id tracks_type = "gt" if is_gt else "pred" @@ -436,74 +437,74 @@ def test_1obj_poor_orientation() -> None: def test_orientation_error1() -> None: """ """ - yaw1 = np.deg2rad(179) - yaw2 = np.deg2rad(-179) + yaw1 = np.deg2rad([179]) + yaw2 = np.deg2rad([-179]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) def test_orientation_error2() -> None: """ """ - yaw1 = np.deg2rad(-179) - yaw2 = np.deg2rad(179) + yaw1 = np.deg2rad([-179]) + yaw2 = np.deg2rad([179]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) print(error_deg) assert np.allclose(error_deg, 2.0, atol=1e-2) def test_orientation_error3() -> None: """ """ - yaw1 = np.deg2rad(179) - yaw2 = np.deg2rad(178) + yaw1 = np.deg2rad([179]) + yaw2 = np.deg2rad([178]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 1.0, atol=1e-2) def test_orientation_error4() -> None: """ """ - yaw1 = np.deg2rad(178) - yaw2 = np.deg2rad(179) + yaw1 = np.deg2rad([178]) + yaw2 = np.deg2rad([179]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 1.0, atol=1e-2) def test_orientation_error5() -> None: """ """ - yaw1 = np.deg2rad(3) - yaw2 = np.deg2rad(-3) + yaw1 = np.deg2rad([3]) + yaw2 = np.deg2rad([-3]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 6.0, atol=1e-2) def test_orientation_error6() -> None: """ """ - yaw1 = np.deg2rad(-3) - yaw2 = np.deg2rad(3) + yaw1 = np.deg2rad([-3]) + yaw2 = np.deg2rad([3]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 6.0, atol=1e-2) def test_orientation_error7() -> None: """ """ - yaw1 = np.deg2rad(-177) - yaw2 = np.deg2rad(-179) + yaw1 = np.deg2rad([-177]) + yaw2 = np.deg2rad([-179]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) def test_orientation_error8() -> None: """ """ - yaw1 = np.deg2rad(-179) - yaw2 = np.deg2rad(-177) + yaw1 = np.deg2rad([-179]) + yaw2 = np.deg2rad([-177]) - error_deg = get_orientation_error_deg(yaw1, yaw2) + error_deg = np.rad2deg(wrap(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) From 97ffb879353427e4097d84ceca88608083d3e838 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Thu, 24 Sep 2020 08:39:30 -0400 Subject: [PATCH 044/113] 'wrap' renamed to 'wrap_angle'. --- argoverse/evaluation/detection_utils.py | 4 ++-- argoverse/evaluation/eval_tracking.py | 6 +++--- tests/test_eval_tracking.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 96eeb152..e3204e80 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -153,7 +153,7 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar gt_quats = np.vstack(gts["quaternion"].array) gt_yaws = R.from_quat(quat_argo2scipy_vectorized(gt_quats)).as_euler("xyz")[:, 2] - orientation_errors = wrap(dt_yaws - gt_yaws) + orientation_errors = wrap_angle(dt_yaws - gt_yaws) return orientation_errors else: raise NotImplemented("This distance metric is not implemented!") @@ -178,7 +178,7 @@ def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: return (inter / union).values -def wrap(angles: np.ndarray, period: float = np.pi) -> np.ndarray: +def wrap_angle(angles: np.ndarray, period: float = np.pi) -> np.ndarray: """Map angles (in radians) from domain [-∞, ∞] to [0, π). This function is the inverse of `np.unwrap`. diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 6a376e49..d7595df8 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -10,13 +10,13 @@ import motmetrics as mm import numpy as np +from click.core import Option from shapely.geometry.polygon import Polygon from argoverse.evaluation.eval_utils import label_to_bbox from argoverse.utils.json_utils import read_json_file -from click.core import Option -from .detection_utils import wrap +from .detection_utils import wrap_angle mh = mm.metrics.create() logger = logging.getLogger(__name__) @@ -98,7 +98,7 @@ def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str return get_distance_iou_3d(x1, x2, name) elif name == "orientation": theta = np.array([x1["orientation"]] - x2["orientation"]) - dist = wrap(theta).item() + dist = wrap_angle(theta).item() # Convert to degrees. return np.rad2deg(dist) diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py index f04021d2..a7ac86ae 100644 --- a/tests/test_eval_tracking.py +++ b/tests/test_eval_tracking.py @@ -9,7 +9,7 @@ import numpy as np from scipy.spatial.transform import Rotation -from argoverse.evaluation.detection_utils import wrap +from argoverse.evaluation.detection_utils import wrap_angle from argoverse.evaluation.eval_tracking import eval_tracks from argoverse.utils.json_utils import save_json_dict @@ -440,7 +440,7 @@ def test_orientation_error1() -> None: yaw1 = np.deg2rad([179]) yaw2 = np.deg2rad([-179]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) @@ -449,7 +449,7 @@ def test_orientation_error2() -> None: yaw1 = np.deg2rad([-179]) yaw2 = np.deg2rad([179]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) print(error_deg) assert np.allclose(error_deg, 2.0, atol=1e-2) @@ -459,7 +459,7 @@ def test_orientation_error3() -> None: yaw1 = np.deg2rad([179]) yaw2 = np.deg2rad([178]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 1.0, atol=1e-2) @@ -468,7 +468,7 @@ def test_orientation_error4() -> None: yaw1 = np.deg2rad([178]) yaw2 = np.deg2rad([179]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 1.0, atol=1e-2) @@ -477,7 +477,7 @@ def test_orientation_error5() -> None: yaw1 = np.deg2rad([3]) yaw2 = np.deg2rad([-3]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 6.0, atol=1e-2) @@ -486,7 +486,7 @@ def test_orientation_error6() -> None: yaw1 = np.deg2rad([-3]) yaw2 = np.deg2rad([3]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 6.0, atol=1e-2) @@ -495,7 +495,7 @@ def test_orientation_error7() -> None: yaw1 = np.deg2rad([-177]) yaw2 = np.deg2rad([-179]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) @@ -504,7 +504,7 @@ def test_orientation_error8() -> None: yaw1 = np.deg2rad([-179]) yaw2 = np.deg2rad([-177]) - error_deg = np.rad2deg(wrap(yaw1 - yaw2)) + error_deg = np.rad2deg(wrap_angle(yaw1 - yaw2)) assert np.allclose(error_deg, 2.0, atol=1e-2) From db2f83fb53c5c35ed1479b0afd1773e7a6ad4a21 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Thu, 24 Sep 2020 12:33:39 -0400 Subject: [PATCH 045/113] Minor cleanup + 'as_dcm' -> 'as_matrix' -- 'as_dcm' is scheduled for deprecation. --- argoverse/utils/transform.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/argoverse/utils/transform.py b/argoverse/utils/transform.py index 8d4c945d..83d89d3b 100644 --- a/argoverse/utils/transform.py +++ b/argoverse/utils/transform.py @@ -26,7 +26,9 @@ def quat2rotmat(q: np.ndarray) -> np.ndarray: R: Array of shape (3, 3) representing a rotation matrix. """ assert np.isclose(np.linalg.norm(q), 1.0, atol=1e-12) - return Rotation.from_quat(quat_argo2scipy(q)).as_dcm() + + quat_xyzw = quat_argo2scipy(q) + return Rotation.from_quat(quat_xyzw).as_matrix() def quat_argo2scipy(q: np.ndarray) -> np.ndarray: diff --git a/setup.py b/setup.py index 62a46bf9..bbd8d285 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ "pillow", "imageio", "pyntcloud @ git+https://github.com/daavoo/pyntcloud#egg=pyntcloud-0.1.0", - "scipy>=1.2.1", + "scipy>=1.4.0", "shapely", "sklearn", "typing_extensions", From a8065699016db8ecdabdb248121c16454c3f07e1 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Thu, 24 Sep 2020 13:08:08 -0400 Subject: [PATCH 046/113] Removed accidental import. --- argoverse/evaluation/eval_tracking.py | 1 - 1 file changed, 1 deletion(-) diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index d7595df8..fd6a0505 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -10,7 +10,6 @@ import motmetrics as mm import numpy as np -from click.core import Option from shapely.geometry.polygon import Polygon from argoverse.evaluation.eval_utils import label_to_bbox From 5078fb93cc8922d98d8fa00f5d94d4531c57b33f Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Thu, 24 Sep 2020 13:14:06 -0400 Subject: [PATCH 047/113] Updated docstrings in 'transform.py' --- argoverse/utils/transform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/argoverse/utils/transform.py b/argoverse/utils/transform.py index 83d89d3b..f34e5dff 100644 --- a/argoverse/utils/transform.py +++ b/argoverse/utils/transform.py @@ -32,17 +32,17 @@ def quat2rotmat(q: np.ndarray) -> np.ndarray: def quat_argo2scipy(q: np.ndarray) -> np.ndarray: - """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" + """Re-order Argoverse's scalar-first [w,x,y,z] quaternion order to Scipy's scalar-last [x,y,z,w]""" w, x, y, z = q q_scipy = np.array([x, y, z, w]) return q_scipy def quat_argo2scipy_vectorized(q: np.ndarray) -> np.ndarray: - """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" + """"Re-order Argoverse's scalar-first [w,x,y,z] quaternion order to Scipy's scalar-last [x,y,z,w]""" return q[..., [1, 2, 3, 0]] def quat_scipy2argo_vectorized(q: np.ndarray) -> np.ndarray: - """Re-order a scalar-first [w, x, y, z] quaternion format used by Argoverse to the scalar-last format in SciPy.""" + """"Re-order Scipy's scalar-last [x,y,z,w] quaternion order to Argoverse's scalar-first [w,x,y,z].""" return q[..., [3, 0, 1, 2]] From 1d90574bba69afb3329f287f65fe1e66c82a64a3 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 24 Sep 2020 19:45:15 +0000 Subject: [PATCH 048/113] Removed '@dataclass' :(. --- argoverse/evaluation/eval_detection.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index caf94aad..241e3ec2 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -58,10 +58,9 @@ import logging import os from collections import defaultdict -from dataclasses import Field, dataclass, field from multiprocessing import Pool from pathlib import Path -from typing import DefaultDict, List, Tuple +from typing import DefaultDict, List, NamedTuple, Tuple import matplotlib import numpy as np @@ -115,8 +114,7 @@ SIGNIFICANT_DIGITS: float = 3 -@dataclass -class DetectionCfg: +class DetectionCfg(NamedTuple): """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. Args: @@ -132,22 +130,18 @@ class DetectionCfg: tp_normalization_terms: Normalization constants for ATE, ASE, and AOE. """ - affinity_threshs: List[float] = field(default_factory=lambda: [0.5, 1.0, 2.0, 4.0]) # Meters + affinity_threshs: List[float] = [0.5, 1.0, 2.0, 4.0] # Meters affinity_fn_type: AffFnType = AffFnType.CENTER n_rec_samples: int = 101 tp_thresh: float = 2.0 # Meters - detection_classes: List[str] = field(default_factory=lambda: list(OBJ_CLASS_MAPPING_DICT.keys())) + detection_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys()) detection_metric: FilterMetric = FilterMetric.EUCLIDEAN max_detection_range: float = 100.0 # Meters save_figs: bool = False - tp_normalization_terms: np.ndarray = field(init=False) + tp_normalization_terms: np.ndarray = np.array([tp_thresh, 1.0, MAX_YAW_ERROR]) - def __post_init__(self): - self.tp_normalization_terms: np.ndarray = np.array([self.tp_thresh, 1.0, MAX_YAW_ERROR]) - -@dataclass -class DetectionEvaluator: +class DetectionEvaluator(NamedTuple): """Instantiates a DetectionEvaluator object for evaluation. Args: From 67f42ec8704d2d38674f87f3ba147beeed797a21 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 24 Sep 2020 18:31:55 -0400 Subject: [PATCH 049/113] add more thorough TP metrics tests --- .../tracked_object_labels_0.json | 2 +- .../tracked_object_labels_1.json | 2 +- .../tracked_object_labels_2.json | 4 +- .../tracked_object_labels_0.json | 42 +++++++++++++++++ .../tracked_object_labels_1.json | 42 +++++++++++++++++ .../tracked_object_labels_2.json | 42 +++++++++++++++++ tests/test_eval_detection.py | 46 ++++++++++++++----- 7 files changed, 164 insertions(+), 16 deletions(-) create mode 100644 tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_0.json create mode 100644 tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_1.json create mode 100644 tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_2.json diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json index 5a63c6f7..32d8a801 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -3,7 +3,7 @@ "center": { "x": 0, "y": 0, - "z": 0 + "z": 0.1 }, "height": 2, "label_class": "VEHICLE", diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json index 63a5c633..ff89c553 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -7,7 +7,7 @@ }, "height": 2, "label_class": "VEHICLE", - "length": 2, + "length": 2.5, "rotation": { "w": 0, "x": 0.0, diff --git a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json index b5f11f76..8edfd180 100644 --- a/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json +++ b/tests/test_data/detection/detections/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -29,9 +29,9 @@ "label_class": "VEHICLE", "length": 2, "rotation": { - "w": 0, + "w": 0.707, "x": 0.0, - "y": 1.0, + "y": 0.707, "z": 0 }, "score": 1.0, diff --git a/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_0.json new file mode 100644 index 00000000..5a63c6f7 --- /dev/null +++ b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -0,0 +1,42 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_1.json new file mode 100644 index 00000000..63a5c633 --- /dev/null +++ b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -0,0 +1,42 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_2.json new file mode 100644 index 00000000..b5f11f76 --- /dev/null +++ b/tests/test_data/detection/detections_identity/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -0,0 +1,42 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 31adba06..44a3cb92 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -20,16 +20,32 @@ @pytest.fixture -def evaluator() -> DetectionEvaluator: +def evaluator_identity() -> DetectionEvaluator: """Define an evaluator that compares a set of results to itself.""" detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) + return DetectionEvaluator( + TEST_DATA_LOC / "detections_identity", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg + ) + + +@pytest.fixture +def evaluator() -> DetectionEvaluator: + """Definte an evaluator that compares a set of detections with known error to the ground truth.""" + detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) +@pytest.fixture +def metrics_identity(evaluator_identity: DetectionEvaluator) -> DataFrame: + """Get the metrics for an evaluator that compares a set of results to itself.""" + return evaluator_identity.evaluate() + + @pytest.fixture def metrics(evaluator: DetectionEvaluator) -> DataFrame: + """Get the metrics for an evaluator with known error.""" return evaluator.evaluate() @@ -141,25 +157,31 @@ def test_iou_aligned_3d() -> None: assert iou_aligned_3d(dt_dims, gt_dims) == expected_result -def test_ap(metrics: DataFrame) -> None: +def test_ap(metrics_identity: DataFrame, metrics: DataFrame) -> None: """Test that AP is 1 for the self-compared results.""" expected_result: float = 1.0 - assert metrics.AP.loc["Average Metrics"] == expected_result + assert metrics_identity.AP.loc["Average Metrics"] == expected_result -def test_translation_error(metrics: DataFrame) -> None: +def test_translation_error(metrics_identity: DataFrame, metrics: DataFrame) -> None: """Test that ATE is 0 for the self-compared results.""" - expected_result: float = 0.0 - assert metrics.ATE.loc["Average Metrics"] == expected_result + expected_result_identity: float = 0.0 + expected_result_det: float = 0.017 # 0.1 / 6, one of six dets is off by 0.1 + assert metrics_identity.ATE.loc["Average Metrics"] == expected_result_identity + assert metrics.ATE.loc["Average Metrics"] == expected_result_det -def test_scale_error(metrics: DataFrame) -> None: +def test_scale_error(metrics_identity: DataFrame, metrics: DataFrame) -> None: """Test that ASE is 0 for the self-compared results.""" - expected_result: float = 0.0 - assert metrics.ASE.loc["Average Metrics"] == expected_result + expected_result_identity: float = 0.0 + expected_result_det: float = 0.033 # 0.2 / 6, one of six dets is off by 20% in IoU + assert metrics_identity.ASE.loc["Average Metrics"] == expected_result_identity + assert metrics.ASE.loc["Average Metrics"] == expected_result_det -def test_orientation_error(metrics: DataFrame) -> None: +def test_orientation_error(metrics_identity: DataFrame, metrics: DataFrame) -> None: """Test that AOE is 0 for the self-compared results.""" - expected_result: float = 0.0 - assert metrics.AOE.loc["Average Metrics"] == expected_result + expected_result_identity: float = 0.0 + expected_result_det: float = 0.524 # pi / 6, since one of six dets is off by pi + assert metrics_identity.AOE.loc["Average Metrics"] == expected_result_identity + assert metrics.AOE.loc["Average Metrics"] == expected_result_det From 27cfeb5e362f21bc5c3d7bf4e8ea0e9ec739092a Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 24 Sep 2020 19:02:17 -0400 Subject: [PATCH 050/113] add assignment test --- .../tracked_object_labels_0.json | 62 +++++++++++++++++++ .../tracked_object_labels_1.json | 42 +++++++++++++ .../tracked_object_labels_2.json | 42 +++++++++++++ tests/test_eval_detection.py | 21 +++++++ 4 files changed, 167 insertions(+) create mode 100644 tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json create mode 100644 tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_1.json create mode 100644 tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_2.json diff --git a/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json new file mode 100644 index 00000000..7edacb67 --- /dev/null +++ b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -0,0 +1,62 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 0, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_1.json b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_1.json new file mode 100644 index 00000000..63a5c633 --- /dev/null +++ b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_1.json @@ -0,0 +1,42 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 0.0, + "z": 1 + }, + "score": 1.0, + "timestamp": 1, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_2.json b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_2.json new file mode 100644 index 00000000..b5f11f76 --- /dev/null +++ b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_2.json @@ -0,0 +1,42 @@ +[ + { + "center": { + "x": 0, + "y": 0, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 1.0, + "y": 0.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000000", + "width": 2 + }, + { + "center": { + "x": 2, + "y": 2, + "z": 0 + }, + "height": 2, + "label_class": "VEHICLE", + "length": 2, + "rotation": { + "w": 0, + "x": 0.0, + "y": 1.0, + "z": 0 + }, + "score": 1.0, + "timestamp": 2, + "track_label_uuid": "00000000-0000-0000-0000-000000000001", + "width": 2 + } +] diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 44a3cb92..cef01652 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -28,6 +28,15 @@ def evaluator_identity() -> DetectionEvaluator: ) +@pytest.fixture +def evaluator_assignment() -> DetectionEvaluator: + """Define an evaluator that compares a set of results to one with an extra detection to check assignment.""" + detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) + return DetectionEvaluator( + TEST_DATA_LOC / "detections_assignment", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg + ) + + @pytest.fixture def evaluator() -> DetectionEvaluator: """Definte an evaluator that compares a set of detections with known error to the ground truth.""" @@ -43,6 +52,12 @@ def metrics_identity(evaluator_identity: DetectionEvaluator) -> DataFrame: return evaluator_identity.evaluate() +@pytest.fixture +def metrics_assignment(evaluator_assignment: DetectionEvaluator) -> DataFrame: + """Get the metrics for an evaluator that has extra detections to test for assignment errors.""" + return evaluator_assignment.evaluate() + + @pytest.fixture def metrics(evaluator: DetectionEvaluator) -> DataFrame: """Get the metrics for an evaluator with known error.""" @@ -157,6 +172,12 @@ def test_iou_aligned_3d() -> None: assert iou_aligned_3d(dt_dims, gt_dims) == expected_result +def test_assignment(metrics_assignment: DataFrame) -> None: + """Verify that assignment works as expected; should have one duplicate in the provided results.""" + expected_result: float = 0.928 + assert metrics_assignment.AP.loc["Average Metrics"] == expected_result + + def test_ap(metrics_identity: DataFrame, metrics: DataFrame) -> None: """Test that AP is 1 for the self-compared results.""" expected_result: float = 1.0 From 96f8d4893e12f94a851d1eb24e7a61fbe0f93f23 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 24 Sep 2020 19:03:33 -0400 Subject: [PATCH 051/113] fix typos --- tests/test_eval_detection.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index cef01652..24e7e900 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -65,7 +65,7 @@ def metrics(evaluator: DetectionEvaluator) -> DataFrame: def test_affinity_center() -> None: - """Intialize a detection and a ground truth label. Verify that calculated distance matches expected affinity + """Initialize a detection and a ground truth label. Verify that calculated distance matches expected affinity under the specified `AffFnType`. """ dts: List[ObjectLabelRecord] = [ @@ -94,7 +94,7 @@ def test_affinity_center() -> None: def test_translation_distance() -> None: - """Intialize a detection and a ground truth label with only translation + """Initialize a detection and a ground truth label with only translation parameters. Verify that calculated distance matches expected distance under the specified `DistFnType`. """ @@ -106,7 +106,7 @@ def test_translation_distance() -> None: def test_scale_distance() -> None: - """Intialize a detection and a ground truth label with only shape + """Initialize a detection and a ground truth label with only shape parameters (only shape parameters due to alignment assumption). Verify that calculated scale error matches the expected value. """ @@ -118,7 +118,7 @@ def test_scale_distance() -> None: def test_orientation_quarter_angles() -> None: - """Intialize a detection and a ground truth label with only orientation + """Initialize a detection and a ground truth label with only orientation parameters. Verify that calculated orientation error matches the expected smallest angle ((2 * np.pi) / 4) between the detection and ground truth label. """ @@ -138,8 +138,8 @@ def test_orientation_quarter_angles() -> None: assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), expected_result) -def test_orientation_eigth_angles() -> None: - """Intialize a detection and a ground truth label with only orientation +def test_orientation_eighth_angles() -> None: + """Initialize a detection and a ground truth label with only orientation parameters. Verify that calculated orientation error matches the expected smallest angle ((2 * np.pi) / 8) between the detection and ground truth label. """ @@ -158,7 +158,7 @@ def test_orientation_eigth_angles() -> None: def test_iou_aligned_3d() -> None: - """Intialize a detection and a ground truth label with only shape + """Initialize a detection and a ground truth label with only shape parameters (only shape parameters due to alignment assumption). Verify that calculated intersection-over-union matches the expected value between the detection and ground truth label. From a62a65ae43e282a71bff743239e5b349673caf3f Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 24 Sep 2020 23:29:41 +0000 Subject: [PATCH 052/113] Updated docstrings. --- argoverse/evaluation/detection_utils.py | 4 ++-- argoverse/evaluation/eval_detection.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index e3204e80..b4d5798a 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -128,8 +128,8 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar """Distance functions between detections and ground truth. Args: - dts: Detections (N, ). - gts: Ground truth labels (N, ). + dts: Detections (N, D) where D is the number of attributes in `ObjectLabelRecord`. + gts: Ground truth labels (N, D) where D is the number of attributes in `ObjectLabelRecord`. metric: Distance function type. Returns: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 241e3ec2..7fdcc8b2 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -296,6 +296,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr dt_tp_indices = unique_dt_matches[tp_mask] gt_tp_indices = unique_gt_matches[tp_mask] + # Form DataFrame of shape (N, D) where D is the number of attributes in `ObjectLabelRecord`. dt_df = pd.DataFrame([dt.__dict__ for dt in dts[dt_tp_indices]]) gt_df = pd.DataFrame([gt.__dict__ for gt in gts[gt_tp_indices]]) From 66c503c9441288258acae60e42ff332607bc7f39 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2020 20:17:03 -0400 Subject: [PATCH 053/113] increase timeout limit for tox --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3c2f93af..1171a969 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - pip install tox-travis==0.12 tox==3.12.1 pre-commit==1.17.0 script: - - tox + - travis_wait 30 tox - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then pre-commit run --show-diff-on-failure --origin $TRAVIS_COMMIT --source $TRAVIS_BRANCH; fi' notifications: From 28a5f660a84b2ea012679c5d91f01dc8e250de50 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 28 Sep 2020 02:26:41 -0400 Subject: [PATCH 054/113] helpful comment --- argoverse/evaluation/eval_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 7fdcc8b2..70c40d27 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -329,7 +329,7 @@ def summarize( for cls_name, cls_stats in data.items(): ninst = cls_to_ninst[cls_name] - ranks = cls_stats[:, -1].argsort()[::-1] + ranks = cls_stats[:, -1].argsort()[::-1] # sort by last column, i.e. confidences cls_stats = cls_stats[ranks] for i, _ in enumerate(self.detection_cfg.affinity_threshs): From 96a0ed4a1fd4d3e706ba8d52545643bfe61121a3 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 28 Sep 2020 02:42:49 -0400 Subject: [PATCH 055/113] move ap to function --- argoverse/evaluation/eval_detection.py | 39 ++++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 70c40d27..54d6a40a 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -309,6 +309,32 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr ).T return metrics + def calc_ap( + self, gts_sorted_by_conf: np.ndarray, recalls_interp: np.ndarray, ninst: int + ) -> Tuple[float, np.ndarray]: + """ Compute precision and recall, interpolated over n fixed recall points. + + Args: + gts_sorted_by_conf: The ground truths, sorted by confidence. + recalls_interp: The interpolated recall values. + ninst: Number of instances of this class. + Returns: + ap_th: The thresholded AP + precisions_interp: The interpolated precision values. + """ + tp = gts_sorted_by_conf + + cumulative_tp = np.cumsum(tp, dtype=np.int) + cumulative_fp = np.cumsum(~tp, dtype=np.int) + cumulative_fn = ninst - cumulative_tp + + precisions = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) + recalls = cumulative_tp / (cumulative_tp + cumulative_fn) + precisions = interp(precisions) + precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) + ap_th = precisions_interp.mean() + return ap_th, precisions_interp + def summarize( self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] ) -> DefaultDict[str, List]: @@ -334,18 +360,7 @@ def summarize( for i, _ in enumerate(self.detection_cfg.affinity_threshs): tp = cls_stats[:, i].astype(bool) - - cumulative_tp = np.cumsum(tp, dtype=np.int) - cumulative_fp = np.cumsum(~tp, dtype=np.int) - cumulative_fn = ninst - cumulative_tp - - precisions = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) - recalls = cumulative_tp / (cumulative_tp + cumulative_fn) - - precisions = interp(precisions) - precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) - - ap_th = precisions_interp.mean() + ap_th, precisions_interp = self.calc_ap(tp, recalls_interp, ninst) summary[cls_name] += [ap_th] if self.detection_cfg.save_figs: From 21187d13fe93b9eb96f62a7673125b09139ada93 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 28 Sep 2020 10:56:27 -0400 Subject: [PATCH 056/113] move calc_ap function to detection_utils --- argoverse/evaluation/detection_utils.py | 25 +++++++++++++++++++++ argoverse/evaluation/eval_detection.py | 29 ++----------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index b4d5798a..c6c5772b 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -124,6 +124,31 @@ def compute_affinity_matrix( return sims +def calc_ap(gts_sorted_by_conf: np.ndarray, recalls_interp: np.ndarray, ninst: int) -> Tuple[float, np.ndarray]: + """ Compute precision and recall, interpolated over n fixed recall points. + + Args: + gts_sorted_by_conf: The ground truths, sorted by confidence. + recalls_interp: The interpolated recall values. + ninst: Number of instances of this class. + Returns: + ap_th: The thresholded AP + precisions_interp: The interpolated precision values. + """ + tp = gts_sorted_by_conf + + cumulative_tp = np.cumsum(tp, dtype=np.int) + cumulative_fp = np.cumsum(~tp, dtype=np.int) + cumulative_fn = ninst - cumulative_tp + + precisions = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) + recalls = cumulative_tp / (cumulative_tp + cumulative_fn) + precisions = interp(precisions) + precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) + ap_th = precisions_interp.mean() + return ap_th, precisions_interp + + def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndarray: """Distance functions between detections and ground truth. diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 54d6a40a..1463cde1 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -73,6 +73,7 @@ AffFnType, DistFnType, FilterMetric, + calc_ap, compute_affinity_matrix, dist_fn, filter_instances, @@ -309,32 +310,6 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr ).T return metrics - def calc_ap( - self, gts_sorted_by_conf: np.ndarray, recalls_interp: np.ndarray, ninst: int - ) -> Tuple[float, np.ndarray]: - """ Compute precision and recall, interpolated over n fixed recall points. - - Args: - gts_sorted_by_conf: The ground truths, sorted by confidence. - recalls_interp: The interpolated recall values. - ninst: Number of instances of this class. - Returns: - ap_th: The thresholded AP - precisions_interp: The interpolated precision values. - """ - tp = gts_sorted_by_conf - - cumulative_tp = np.cumsum(tp, dtype=np.int) - cumulative_fp = np.cumsum(~tp, dtype=np.int) - cumulative_fn = ninst - cumulative_tp - - precisions = cumulative_tp / (cumulative_tp + cumulative_fp + np.finfo(float).eps) - recalls = cumulative_tp / (cumulative_tp + cumulative_fn) - precisions = interp(precisions) - precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) - ap_th = precisions_interp.mean() - return ap_th, precisions_interp - def summarize( self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] ) -> DefaultDict[str, List]: @@ -360,7 +335,7 @@ def summarize( for i, _ in enumerate(self.detection_cfg.affinity_threshs): tp = cls_stats[:, i].astype(bool) - ap_th, precisions_interp = self.calc_ap(tp, recalls_interp, ninst) + ap_th, precisions_interp = calc_ap(tp, recalls_interp, ninst) summary[cls_name] += [ap_th] if self.detection_cfg.save_figs: From 05819e0e63f8d2f8e7d45e17f42860fb1010cd60 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Mon, 28 Sep 2020 11:53:54 -0400 Subject: [PATCH 057/113] Small docstring updates. --- argoverse/evaluation/detection_utils.py | 42 ++++++++++++------------- argoverse/evaluation/eval_detection.py | 14 ++++----- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index c6c5772b..66546c80 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -46,13 +46,13 @@ def filter_instances( """Filter the GT annotations based on a set of conditions (class name and distance from egovehicle). Args: - instances: The instances to be filtered (N, ). - target_class_name: The name of the class of interest. - filter_metric: The range metric used for filtering. - max_detection_range: The maximum distance for range filtering. + instances: Instances to be filtered (N, ). + target_class_name: Name of the class of interest. + filter_metric: Range metric used for filtering. + max_detection_range: Maximum distance for range filtering. Returns: - The filtered annotations. + Filtered annotations. """ instances = np.array([instance for instance in instances if instance.label_class == target_class_name]) @@ -75,8 +75,8 @@ def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: dts: Detections (N,). Returns: - ranks: The ranking for the detections (N, ). - scores: The detection scores (N, ). + ranks: Ranking for the detections (N, ). + scores: Detection scores (N, ). """ scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] @@ -92,7 +92,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: method: Accumulation method. Returns: - prec_interp: Interpolated precision at all recall levels (N,). + prec_interp: Interpolated precision at all recall levels (N, ). """ if method == InterpType.ALL: prec_interp = np.maximum.accumulate(prec[::-1])[::-1] @@ -104,16 +104,16 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: def compute_affinity_matrix( dts: List[ObjectLabelRecord], gts: List[ObjectLabelRecord], metric: AffFnType ) -> np.ndarray: - """Calculate the match matrix between detections and ground truth labels, - using a specified affinity function. + """Calculate the affinity matrix between detections and ground truth labels, + using a specified affinity function type. Args: dts: Detections (N, ). gts: Ground truth labels (M, ). - metric: Similarity metric type. + metric: Affinity metric type. Returns: - sims: Similarity scores between detections and ground truth annotations (N, M). + sims: Affinity scores between detections and ground truth annotations (N, M). """ if metric == AffFnType.CENTER: dt_centers = np.array([dt.translation for dt in dts]) @@ -124,18 +124,18 @@ def compute_affinity_matrix( return sims -def calc_ap(gts_sorted_by_conf: np.ndarray, recalls_interp: np.ndarray, ninst: int) -> Tuple[float, np.ndarray]: - """ Compute precision and recall, interpolated over n fixed recall points. +def calc_ap(gt_ranked: np.ndarray, recalls_interp: np.ndarray, ninst: int) -> Tuple[float, np.ndarray]: + """Compute precision and recall, interpolated over n fixed recall points. Args: - gts_sorted_by_conf: The ground truths, sorted by confidence. - recalls_interp: The interpolated recall values. + gt_ranked: Ground truths, ranked by confidence. + recalls_interp: Interpolated recall values. ninst: Number of instances of this class. Returns: - ap_th: The thresholded AP - precisions_interp: The interpolated precision values. + avg_precision: Average precision. + precisions_interp: Interpolated precision values. """ - tp = gts_sorted_by_conf + tp = gt_ranked cumulative_tp = np.cumsum(tp, dtype=np.int) cumulative_fp = np.cumsum(~tp, dtype=np.int) @@ -145,8 +145,8 @@ def calc_ap(gts_sorted_by_conf: np.ndarray, recalls_interp: np.ndarray, ninst: i recalls = cumulative_tp / (cumulative_tp + cumulative_fn) precisions = interp(precisions) precisions_interp = np.interp(recalls_interp, recalls, precisions, right=0) - ap_th = precisions_interp.mean() - return ap_th, precisions_interp + avg_precision = precisions_interp.mean() + return avg_precision, precisions_interp def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndarray: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 1463cde1..7e434855 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -163,7 +163,7 @@ def evaluate(self) -> pd.DataFrame: annotations. Returns: - Evaluation metrics of shape (C + 1, K) where C + 1 is the number of classes + Evaluation metrics of shape (C + 1, K) where C + 1 is the number of classes. plus a row for their means. K refers to the number of evaluation metrics. """ dt_fpaths = list(self.detection_fpath.glob("*/per_sweep_annotations_amodal/*.json")) @@ -255,7 +255,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr Returns: metrics: Matrix of true/false positive concatenated with true positive errors (N, K + S) where K is the number of true positive thresholds used for AP computation and S is the number of true positive errors. - scores: Corresponding scores for the true positives/false positives (N,) + scores: Corresponding scores for the true positives/false positives (N,). """ # Ensure the number of boxes considered per class is at most `MAX_NUM_BOXES`. @@ -284,7 +284,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) for i, thresh in enumerate(self.detection_cfg.affinity_threshs): - # `tp_mask` may need to be defined differently with other affinities + # `tp_mask` may need to be defined differently with other affinities. tp_mask = affinities[unique_dt_matches] > -thresh metrics[unique_dt_matches, i] = tp_mask @@ -341,23 +341,23 @@ def summarize( if self.detection_cfg.save_figs: self.plot(recalls_interp, precisions_interp, cls_name) - # AP Metric + # AP Metric. ap = np.array(summary[cls_name][:num_ths]).mean() # Select only the true positives for each instance. tp_metrics_mask = ~np.isnan(cls_stats[:, num_ths : num_ths + N_TP_ERRORS]).all(axis=1) - # If there are no true positives set tp errors to their maximum values due to normalization below) + # If there are no true positives set tp errors to their maximum values due to normalization below). if ~tp_metrics_mask.any(): tp_metrics = self.detection_cfg.tp_normalization_terms else: # Calculate TP metrics. tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + N_TP_ERRORS][tp_metrics_mask], axis=0) - # Convert errors to scores + # Convert errors to scores. tp_scores = 1 - (tp_metrics / self.detection_cfg.tp_normalization_terms) - # Compute Composite Detection Score (CDS) + # Compute Composite Detection Score (CDS). cds = ap * tp_scores.mean() summary[cls_name] = [ap, *tp_metrics, cds] From 92313780bd1421ec30fd331f5c6f4a4b1a15f82d Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Mon, 28 Sep 2020 12:03:48 -0400 Subject: [PATCH 058/113] Fixed tuple styling in docstrings. --- argoverse/evaluation/detection_utils.py | 18 +++++++++--------- argoverse/evaluation/eval_detection.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 66546c80..3224305c 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -46,7 +46,7 @@ def filter_instances( """Filter the GT annotations based on a set of conditions (class name and distance from egovehicle). Args: - instances: Instances to be filtered (N, ). + instances: Instances to be filtered (N,). target_class_name: Name of the class of interest. filter_metric: Range metric used for filtering. max_detection_range: Maximum distance for range filtering. @@ -75,8 +75,8 @@ def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]: dts: Detections (N,). Returns: - ranks: Ranking for the detections (N, ). - scores: Detection scores (N, ). + ranks: Ranking for the detections (N,). + scores: Detection scores (N,). """ scores = np.array([dt.score for dt in dts]) ranks = scores.argsort()[::-1] @@ -88,11 +88,11 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: """Interpolate the precision over all recall levels. Args: - prec: Precision at all recall levels (N, ). + prec: Precision at all recall levels (N,). method: Accumulation method. Returns: - prec_interp: Interpolated precision at all recall levels (N, ). + prec_interp: Interpolated precision at all recall levels (N,). """ if method == InterpType.ALL: prec_interp = np.maximum.accumulate(prec[::-1])[::-1] @@ -108,8 +108,8 @@ def compute_affinity_matrix( using a specified affinity function type. Args: - dts: Detections (N, ). - gts: Ground truth labels (M, ). + dts: Detections (N,). + gts: Ground truth labels (M,). metric: Affinity metric type. Returns: @@ -158,7 +158,7 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar metric: Distance function type. Returns: - Distance between the detections and ground truth, using the provided metric (N, ). + Distance between the detections and ground truth, using the provided metric (N,). """ if metric == DistFnType.TRANSLATION: dt_centers = np.vstack(dts["translation"].array) @@ -195,7 +195,7 @@ def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: Returns: Intersection-over-union between the detections and their assigned ground - truth labels (N, ). + truth labels (N,). """ inter = np.minimum(dt_dims, gt_dims).prod(axis=1) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 7e434855..7324cb4b 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -207,7 +207,7 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa cls_to_accum: Class to accumulated statistics dictionary of shape |C| -> (N, K + S) where C is the number of detection classes, K is the number of true positive thresholds used for AP computation, and S is the number of true positive errors. - cls_to_ninst: Mapping of shape |C| -> (1, ) the class names to the number of instances in the ground + cls_to_ninst: Mapping of shape |C| -> (1,) the class names to the number of instances in the ground truth dataset. """ log_id = gt_fpath.parents[1].stem From de025a1feaa408ce99083cffbf6124d9eafc9a6d Mon Sep 17 00:00:00 2001 From: Sean Date: Tue, 29 Sep 2020 15:46:06 -0400 Subject: [PATCH 059/113] fix score ordering in assignment test --- .../per_sweep_annotations_amodal/tracked_object_labels_0.json | 4 ++-- tests/test_eval_detection.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json index 7edacb67..8b581d06 100644 --- a/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json +++ b/tests/test_data/detection/detections_assignment/1/per_sweep_annotations_amodal/tracked_object_labels_0.json @@ -34,7 +34,7 @@ "y": 0.0, "z": 1 }, - "score": 1.0, + "score": 0.9, "timestamp": 0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "width": 2 @@ -54,7 +54,7 @@ "y": 0.0, "z": 1 }, - "score": 1.0, + "score": 0.9, "timestamp": 0, "track_label_uuid": "00000000-0000-0000-0000-000000000001", "width": 2 diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 24e7e900..5755517c 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -174,7 +174,7 @@ def test_iou_aligned_3d() -> None: def test_assignment(metrics_assignment: DataFrame) -> None: """Verify that assignment works as expected; should have one duplicate in the provided results.""" - expected_result: float = 0.928 + expected_result: float = 0.976 assert metrics_assignment.AP.loc["Average Metrics"] == expected_result From f8fe26bba5f13d973fce781dd35a13bcf2aa8baa Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 29 Sep 2020 20:45:16 +0000 Subject: [PATCH 060/113] Made naming consistent + fixed printed default errors when no tp. --- argoverse/evaluation/eval_detection.py | 87 ++++++++++++-------------- tests/test_eval_detection.py | 6 +- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 7324cb4b..3510ddb4 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -95,6 +95,7 @@ STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] +MAX_SCALE_ERROR: float = 1.0 MAX_YAW_ERROR: float = np.pi # Higher is better. @@ -106,9 +107,6 @@ MAX_NORMALIZED_ASE: float = 1.0 MAX_NORMALIZED_AOE: float = 1.0 -# Each measure is in [0, 1]. -MEASURE_DEFAULT_VALUES: List[float] = [MIN_AP, MAX_NORMALIZED_ATE, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS] - # Max number of boxes considered per class per scene. MAX_NUM_BOXES: int = 500 @@ -119,43 +117,45 @@ class DetectionCfg(NamedTuple): """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. Args: - affinity_threshs: The affinity thresholds for determining a true positive. - affinity_fn_type: The type of affinity function to be used for calculating average precision. + affinity_threshs: Affinity thresholds for determining a true positive. + affinity_fn_type: Type of affinity function to be used for calculating average precision. n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. tp_thresh: Center distance threshold for the true positive metrics (in meters). - detection_classes: Detection classes for evaluation. - detection_metric: The detection metric to use for filtering of both detections and ground truth annotations. - max_detection_range: The max distance (under a specific metric in meters) for a detection or ground truth to be + dt_classes: Detection classes for evaluation. + dt_metric: Detection metric to use for filtering of both detections and ground truth annotations. + max_dt_range: The max distance (under a specific metric in meters) for a detection or ground truth to be considered for evaluation. save_figs: Flag to save figures. tp_normalization_terms: Normalization constants for ATE, ASE, and AOE. + summary_default_vals: Evaluation summary default values. """ affinity_threshs: List[float] = [0.5, 1.0, 2.0, 4.0] # Meters affinity_fn_type: AffFnType = AffFnType.CENTER n_rec_samples: int = 101 tp_thresh: float = 2.0 # Meters - detection_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys()) - detection_metric: FilterMetric = FilterMetric.EUCLIDEAN - max_detection_range: float = 100.0 # Meters + dt_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys()) + dt_metric: FilterMetric = FilterMetric.EUCLIDEAN + max_dt_range: float = 100.0 # Meters save_figs: bool = False - tp_normalization_terms: np.ndarray = np.array([tp_thresh, 1.0, MAX_YAW_ERROR]) + tp_normalization_terms: np.ndarray = np.array([tp_thresh, MAX_SCALE_ERROR, MAX_YAW_ERROR]) + summary_default_vals: np.ndarray = np.array([MIN_AP, tp_thresh, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS]) class DetectionEvaluator(NamedTuple): """Instantiates a DetectionEvaluator object for evaluation. Args: - detection_fpath: The path to the folder which contains the detections. - ground_truth_fpath: The path to the folder which contains all the logs. - figures_fpath: The path to the folder which will contain the output figures. - detection_cfg: The detection configuration settings. + dt_fpath_root: Path to the folder which contains the detections. + gt_fpath_root: Path to the folder which contains the split of logs. + figs_fpath: Path to the folder which will contain the output figures. + cfg: Detection configuration settings. """ - detection_fpath: Path - ground_truth_fpath: Path - figures_fpath: Path - detection_cfg: DetectionCfg = DetectionCfg() + dt_root_fpath: Path + gt_root_fpath: Path + figs_fpath: Path + cfg: DetectionCfg = DetectionCfg() def evaluate(self) -> pd.DataFrame: """Evaluate detection output and return metrics. The multiprocessing @@ -166,8 +166,8 @@ def evaluate(self) -> pd.DataFrame: Evaluation metrics of shape (C + 1, K) where C + 1 is the number of classes. plus a row for their means. K refers to the number of evaluation metrics. """ - dt_fpaths = list(self.detection_fpath.glob("*/per_sweep_annotations_amodal/*.json")) - gt_fpaths = list(self.ground_truth_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + dt_fpaths = list(self.dt_root_fpath.glob("*/per_sweep_annotations_amodal/*.json")) + gt_fpaths = list(self.gt_root_fpath.glob("*/per_sweep_annotations_amodal/*.json")) assert len(dt_fpaths) == len(gt_fpaths) data: DefaultDict[str, np.ndarray] = defaultdict(list) @@ -184,7 +184,7 @@ def evaluate(self) -> pd.DataFrame: data = defaultdict(np.ndarray, {k: np.vstack(v) for k, v in data.items()}) - init_data = {dt_cls: MEASURE_DEFAULT_VALUES for dt_cls in self.detection_cfg.detection_classes} + init_data = {dt_cls: self.cfg.summary_default_vals for dt_cls in self.cfg.dt_classes} summary = pd.DataFrame.from_dict(init_data, orient="index", columns=STATISTIC_NAMES) summary_update = pd.DataFrame.from_dict( self.summarize(data, cls_to_ninst), orient="index", columns=STATISTIC_NAMES @@ -214,25 +214,19 @@ def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], Defa logger.info(f"log_id = {log_id}") ts = gt_fpath.stem.split("_")[-1] - dt_fpath = self.detection_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" + dt_fpath = self.dt_root_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" dts = np.array(read_label(str(dt_fpath))) gts = np.array(read_label(str(gt_fpath))) cls_to_accum = defaultdict(list) cls_to_ninst = defaultdict(int) - for class_name in self.detection_cfg.detection_classes: + for class_name in self.cfg.dt_classes: dt_filtered = filter_instances( - dts, - class_name, - filter_metric=self.detection_cfg.detection_metric, - max_detection_range=self.detection_cfg.max_detection_range, + dts, class_name, filter_metric=self.cfg.dt_metric, max_detection_range=self.cfg.max_dt_range ) gt_filtered = filter_instances( - gts, - class_name, - filter_metric=self.detection_cfg.detection_metric, - max_detection_range=self.detection_cfg.max_detection_range, + gts, class_name, filter_metric=self.cfg.dt_metric, max_detection_range=self.cfg.max_dt_range ) logger.info(f"{dt_filtered.shape[0]} detections") @@ -262,7 +256,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr if dts.shape[0] > MAX_NUM_BOXES: dts = dts[:MAX_NUM_BOXES] - n_threshs = len(self.detection_cfg.affinity_threshs) + n_threshs = len(self.cfg.affinity_threshs) metrics = np.zeros((dts.shape[0], n_threshs + N_TP_ERRORS)) # Set the true positive metrics to np.nan since error is undefined on false positives. @@ -270,7 +264,7 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr if gts.shape[0] == 0: return metrics - affinity_matrix = compute_affinity_matrix(dts, gts, self.detection_cfg.affinity_fn_type) + affinity_matrix = compute_affinity_matrix(dts, gts, self.cfg.affinity_fn_type) # Get the GT label for each max-affinity GT label, detection pair. gt_matches = affinity_matrix.argmax(axis=1)[np.newaxis, :] @@ -282,14 +276,14 @@ def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarr # Find the indices of the "first" detection assigned to each GT. unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) - for i, thresh in enumerate(self.detection_cfg.affinity_threshs): + for i, thresh in enumerate(self.cfg.affinity_threshs): # `tp_mask` may need to be defined differently with other affinities. tp_mask = affinities[unique_dt_matches] > -thresh metrics[unique_dt_matches, i] = tp_mask # Only compute true positive error when `thresh` is equal to the tp threshold. - is_tp_thresh = thresh == self.detection_cfg.tp_thresh + is_tp_thresh = thresh == self.cfg.tp_thresh # Ensure that there are true positives of the respective class in the frame. has_true_positives = np.count_nonzero(tp_mask) > 0 @@ -323,22 +317,22 @@ def summarize( summary: The summary statistics. """ summary: DefaultDict[str, List] = defaultdict(list) - recalls_interp = np.linspace(0, 1, self.detection_cfg.n_rec_samples) - num_ths = len(self.detection_cfg.affinity_threshs) - if not self.figures_fpath.is_dir(): - self.figures_fpath.mkdir(parents=True, exist_ok=True) + recalls_interp = np.linspace(0, 1, self.cfg.n_rec_samples) + num_ths = len(self.cfg.affinity_threshs) + if not self.figs_fpath.is_dir(): + self.figs_fpath.mkdir(parents=True, exist_ok=True) for cls_name, cls_stats in data.items(): ninst = cls_to_ninst[cls_name] ranks = cls_stats[:, -1].argsort()[::-1] # sort by last column, i.e. confidences cls_stats = cls_stats[ranks] - for i, _ in enumerate(self.detection_cfg.affinity_threshs): + for i, _ in enumerate(self.cfg.affinity_threshs): tp = cls_stats[:, i].astype(bool) ap_th, precisions_interp = calc_ap(tp, recalls_interp, ninst) summary[cls_name] += [ap_th] - if self.detection_cfg.save_figs: + if self.cfg.save_figs: self.plot(recalls_interp, precisions_interp, cls_name) # AP Metric. @@ -349,13 +343,14 @@ def summarize( # If there are no true positives set tp errors to their maximum values due to normalization below). if ~tp_metrics_mask.any(): - tp_metrics = self.detection_cfg.tp_normalization_terms + tp_metrics = self.cfg.tp_normalization_terms else: # Calculate TP metrics. tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + N_TP_ERRORS][tp_metrics_mask], axis=0) # Convert errors to scores. - tp_scores = 1 - (tp_metrics / self.detection_cfg.tp_normalization_terms) + tp_scores = 1 - (tp_metrics / self.cfg.tp_normalization_terms) + print((tp_metrics / self.cfg.tp_normalization_terms)) # Compute Composite Detection Score (CDS). cds = ap * tp_scores.mean() @@ -377,7 +372,7 @@ def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) - plt.title("PR Curve") plt.xlabel("Recall") plt.ylabel("Precision") - plt.savefig(f"{self.figures_fpath}/{cls_name}.png") + plt.savefig(f"{self.figs_fpath}/{cls_name}.png") plt.close() diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py index 5755517c..3150aac0 100644 --- a/tests/test_eval_detection.py +++ b/tests/test_eval_detection.py @@ -22,7 +22,7 @@ @pytest.fixture def evaluator_identity() -> DetectionEvaluator: """Define an evaluator that compares a set of results to itself.""" - detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) + detection_cfg = DetectionCfg(dt_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections_identity", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) @@ -31,7 +31,7 @@ def evaluator_identity() -> DetectionEvaluator: @pytest.fixture def evaluator_assignment() -> DetectionEvaluator: """Define an evaluator that compares a set of results to one with an extra detection to check assignment.""" - detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) + detection_cfg = DetectionCfg(dt_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections_assignment", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) @@ -40,7 +40,7 @@ def evaluator_assignment() -> DetectionEvaluator: @pytest.fixture def evaluator() -> DetectionEvaluator: """Definte an evaluator that compares a set of detections with known error to the ground truth.""" - detection_cfg = DetectionCfg(detection_classes=["VEHICLE"]) + detection_cfg = DetectionCfg(dt_classes=["VEHICLE"]) return DetectionEvaluator( TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg ) From d9d0ee95b65f5f35d27f66896db35d4c31d96f27 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 11:24:13 -0400 Subject: [PATCH 061/113] fix numerical precision check --- tests/test_frustum_clipping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index d6f5b81b..7b357b8c 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -53,7 +53,7 @@ def test_fit_plane_to_point_cloud() -> None: pc = np.array([[0, 2, 1], [1, 2, 1], [0, 0, 0]]) a, b, c, d = fit_plane_to_point_cloud(pc) - assert d == 0 # touching origin + assert np.isclose(d, 0) # touching origin normal = np.array([a, b, c]) normal /= np.linalg.norm(normal) gt_normal = np.array([0.0, -1.0, 2.0]) From 28a35c6206a0974e81bf6de097baa249ec3f118b Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 12:14:21 -0400 Subject: [PATCH 062/113] re-format for black --- tests/test_frustum_clipping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index 7b357b8c..cf537d6b 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -53,7 +53,7 @@ def test_fit_plane_to_point_cloud() -> None: pc = np.array([[0, 2, 1], [1, 2, 1], [0, 0, 0]]) a, b, c, d = fit_plane_to_point_cloud(pc) - assert np.isclose(d, 0) # touching origin + assert np.isclose(d, 0) # touching origin normal = np.array([a, b, c]) normal /= np.linalg.norm(normal) gt_normal = np.array([0.0, -1.0, 2.0]) From e87247141f74ba77bc3efdfdab628439aed7ed2f Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 12:18:54 -0400 Subject: [PATCH 063/113] fix return type to include Optional --- argoverse/utils/frustum_clipping.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/argoverse/utils/frustum_clipping.py b/argoverse/utils/frustum_clipping.py index 67c9723a..c23c5cff 100644 --- a/argoverse/utils/frustum_clipping.py +++ b/argoverse/utils/frustum_clipping.py @@ -251,7 +251,11 @@ def generate_frustum_planes(K: np.ndarray, camera_name: str, near_clip_dist: flo return planes -def clip_segment_v3_plane_n(p1: np.ndarray, p2: np.ndarray, planes: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]: +def clip_segment_v3_plane_n( + p1: np.ndarray, + p2: np.ndarray, + planes: List[np.ndarray] +) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: """Iterate over the frustum planes and intersect them with the segment. This updating the min/max, bailing out early if the min > max. From 95b9be5715b2889fb3679a09158fbec0d988effb Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 13:08:42 -0400 Subject: [PATCH 064/113] reformat for python black --- argoverse/utils/frustum_clipping.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/argoverse/utils/frustum_clipping.py b/argoverse/utils/frustum_clipping.py index c23c5cff..f54c3564 100644 --- a/argoverse/utils/frustum_clipping.py +++ b/argoverse/utils/frustum_clipping.py @@ -252,9 +252,7 @@ def generate_frustum_planes(K: np.ndarray, camera_name: str, near_clip_dist: flo def clip_segment_v3_plane_n( - p1: np.ndarray, - p2: np.ndarray, - planes: List[np.ndarray] + p1: np.ndarray, p2: np.ndarray, planes: List[np.ndarray] ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: """Iterate over the frustum planes and intersect them with the segment. From c8ff45fb6a50e0a6e1f1355a9ce9aafdfa8dfcf3 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 13:29:50 -0400 Subject: [PATCH 065/113] fix img width, height to be integers for mypy in unit test --- tests/test_frustum_clipping.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index cf537d6b..4e2a5d54 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -167,7 +167,7 @@ def test_clip_segment_v3_plane_n() -> None: def test_form_right_clipping_plane() -> None: """Test form_right_clipping_plane(). Use 4 points to fit the right clipping plane.""" fx = 10.0 - img_width = 30.0 + img_width = 30 right_plane = form_right_clipping_plane(fx, img_width) Y_OFFSET = 10 # arbitrary extent down the imager @@ -187,7 +187,7 @@ def test_form_right_clipping_plane() -> None: def test_form_left_clipping_plane() -> None: """Test form_left_clipping_plane(). Use 4 points to fit the left clipping plane.""" fx = 10.0 - img_width = 30.0 + img_width = 30 left_plane = form_left_clipping_plane(fx, img_width) Y_OFFSET = 10 @@ -206,7 +206,7 @@ def test_form_left_clipping_plane() -> None: def test_form_top_clipping_plane() -> None: """Test form_top_clipping_plane(). Use 3 points to fit the TOP clipping plane.""" fx = 10.0 - img_height = 45.0 + img_height = 45 top_plane = form_top_clipping_plane(fx, img_height) img_width = 1000.0 @@ -226,7 +226,7 @@ def test_form_top_clipping_plane() -> None: def test_form_low_clipping_plane() -> None: """Test form_low_clipping_plane().""" fx = 12.0 - img_height = 35.0 + img_height = 35 low_plane = form_low_clipping_plane(fx, img_height) img_width = 10000 @@ -246,8 +246,8 @@ def test_form_low_clipping_plane() -> None: def test_form_near_clipping_plane() -> None: """Test form_near_clipping_plane(). Use 4 points to fit the near clipping plane.""" - img_width = 10.0 - img_height = 15.0 + img_width = 10 + img_height = 15 near_clip_dist = 30.0 near_plane = form_near_clipping_plane(near_clip_dist) From 252aec8b88d355146e32f49547f9760fac62291a Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 14:30:27 -0400 Subject: [PATCH 066/113] reformat according to python black format --- tests/test_frustum_clipping.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index 4e2a5d54..2abd4831 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -288,8 +288,9 @@ def test_generate_frustum_planes_ring_cam() -> None: camera_name = "ring_front_right" img_height = 1200 img_width = 1920 - planes = generate_frustum_planes(K, camera_name, near_clip_dist=near_clip_dist) - [left_plane, right_plane, near_plane, low_plane, top_plane] = planes + left_plane, right_plane, near_plane, low_plane, top_plane = generate_frustum_planes( + K, camera_name, near_clip_dist=near_clip_dist + ) fx = K[0, 0] left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0]) @@ -327,9 +328,10 @@ def test_generate_frustum_planes_stereo() -> None: camera_name = "stereo_front_left" img_height = 2056 img_width = 2464 - planes = generate_frustum_planes(K, camera_name, near_clip_dist=near_clip_dist) - [left_plane, right_plane, near_plane, low_plane, top_plane] = planes - + left_plane, right_plane, near_plane, low_plane, top_plane = generate_frustum_planes( + K, camera_name, near_clip_dist=near_clip_dist + ) + fx = K[0, 0] left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0]) right_plane_gt = np.array([-fx, 0.0, img_width / 2.0, 0.0]) From 687d12bab016e56f15918bb2e8bc43f1d5be831f Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2020 15:31:39 -0400 Subject: [PATCH 067/113] check for None per mypy spec --- tests/test_frustum_clipping.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index 2abd4831..71f22398 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -288,9 +288,10 @@ def test_generate_frustum_planes_ring_cam() -> None: camera_name = "ring_front_right" img_height = 1200 img_width = 1920 - left_plane, right_plane, near_plane, low_plane, top_plane = generate_frustum_planes( - K, camera_name, near_clip_dist=near_clip_dist - ) + planes = generate_frustum_planes(K, camera_name, near_clip_dist=near_clip_dist) + if planes is None: + assert False + left_plane, right_plane, near_plane, low_plane, top_plane = planes fx = K[0, 0] left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0]) @@ -328,9 +329,10 @@ def test_generate_frustum_planes_stereo() -> None: camera_name = "stereo_front_left" img_height = 2056 img_width = 2464 - left_plane, right_plane, near_plane, low_plane, top_plane = generate_frustum_planes( - K, camera_name, near_clip_dist=near_clip_dist - ) + planes = generate_frustum_planes(K, camera_name, near_clip_dist=near_clip_dist) + if planes is None: + assert False + left_plane, right_plane, near_plane, low_plane, top_plane = planes fx = K[0, 0] left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0]) From 995a2350c590fe0815a9a0e010d20f0290e72b5e Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:17:46 -0400 Subject: [PATCH 068/113] Made a few detection methods functional. --- argoverse/evaluation/detection_utils.py | 174 +++++++++++++++++++++++- argoverse/evaluation/eval_detection.py | 169 +---------------------- 2 files changed, 176 insertions(+), 167 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 3224305c..5128e963 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -10,17 +10,46 @@ """ +import logging +from collections import defaultdict from enum import Enum, auto -from typing import List, Tuple +from pathlib import Path +from typing import DefaultDict, List, NamedTuple, Tuple import numpy as np import pandas as pd from scipy.spatial.distance import cdist from scipy.spatial.transform import Rotation as R -from argoverse.data_loading.object_label_record import ObjectLabelRecord +from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT +from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label from argoverse.utils.transform import quat_argo2scipy_vectorized +logger = logging.getLogger(__name__) + + +TP_ERROR_NAMES: List[str] = ["ATE", "ASE", "AOE"] +N_TP_ERRORS: int = len(TP_ERROR_NAMES) + +STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] + +MAX_SCALE_ERROR: float = 1.0 +MAX_YAW_ERROR: float = np.pi + +# Higher is better. +MIN_AP: float = 0.0 +MIN_CDS: float = 0.0 + +# Lower is better. +MAX_NORMALIZED_ATE: float = 1.0 +MAX_NORMALIZED_ASE: float = 1.0 +MAX_NORMALIZED_AOE: float = 1.0 + +# Max number of boxes considered per class per scene. +MAX_NUM_BOXES: int = 500 + +SIGNIFICANT_DIGITS: float = 3 + class AffFnType(Enum): CENTER = auto() @@ -40,6 +69,147 @@ class FilterMetric(Enum): EUCLIDEAN = auto() +class DetectionCfg(NamedTuple): + """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. + + Args: + affinity_threshs: Affinity thresholds for determining a true positive. + affinity_fn_type: Type of affinity function to be used for calculating average precision. + n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. + tp_thresh: Center distance threshold for the true positive metrics (in meters). + dt_classes: Detection classes for evaluation. + dt_metric: Detection metric to use for filtering of both detections and ground truth annotations. + max_dt_range: The max distance (under a specific metric in meters) for a detection or ground truth to be + considered for evaluation. + save_figs: Flag to save figures. + tp_normalization_terms: Normalization constants for ATE, ASE, and AOE. + summary_default_vals: Evaluation summary default values. + """ + + affinity_threshs: List[float] = [0.5, 1.0, 2.0, 4.0] # Meters + affinity_fn_type: AffFnType = AffFnType.CENTER + n_rec_samples: int = 101 + tp_thresh: float = 2.0 # Meters + dt_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys()) + dt_metric: FilterMetric = FilterMetric.EUCLIDEAN + max_dt_range: float = 100.0 # Meters + save_figs: bool = False + tp_normalization_terms: np.ndarray = np.array([tp_thresh, MAX_SCALE_ERROR, MAX_YAW_ERROR]) + summary_default_vals: np.ndarray = np.array([MIN_AP, tp_thresh, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS]) + + +def accumulate( + dt_root_fpath: Path, gt_fpath: Path, cfg: DetectionCfg +) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: + """Accumulate the true/false positives (boolean flags) and true positive errors for each class. + + Args: + gt_fpath: Ground truth file path. + + Returns: + cls_to_accum: Class to accumulated statistics dictionary of shape |C| -> (N, K + S) where C + is the number of detection classes, K is the number of true positive thresholds used for + AP computation, and S is the number of true positive errors. + cls_to_ninst: Mapping of shape |C| -> (1,) the class names to the number of instances in the ground + truth dataset. + """ + log_id = gt_fpath.parents[1].stem + logger.info(f"log_id = {log_id}") + ts = gt_fpath.stem.split("_")[-1] + + dt_fpath = dt_root_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" + + dts = np.array(read_label(str(dt_fpath))) + gts = np.array(read_label(str(gt_fpath))) + + cls_to_accum = defaultdict(list) + cls_to_ninst = defaultdict(int) + for class_name in cfg.dt_classes: + dt_filtered = filter_instances( + dts, class_name, filter_metric=cfg.dt_metric, max_detection_range=cfg.max_dt_range + ) + gt_filtered = filter_instances( + gts, class_name, filter_metric=cfg.dt_metric, max_detection_range=cfg.max_dt_range + ) + + logger.info(f"{dt_filtered.shape[0]} detections") + logger.info(f"{gt_filtered.shape[0]} ground truth") + if dt_filtered.shape[0] > 0: + ranked_detections, scores = rank(dt_filtered) + metrics = assign(ranked_detections, gt_filtered, cfg) + cls_to_accum[class_name] = np.hstack((metrics, scores)) + + cls_to_ninst[class_name] = gt_filtered.shape[0] + return cls_to_accum, cls_to_ninst + + +def assign(dts: np.ndarray, gts: np.ndarray, cfg: DetectionCfg) -> Tuple[np.ndarray, np.ndarray]: + """Attempt assignment of each detection to a ground truth label. + + Args: + dts: Detections of shape (N,). + gts: Ground truth labels of shape (M,). + cfg: Detection configuration. + + Returns: + metrics: Matrix of true/false positive concatenated with true positive errors (N, K + S) where K is the number + of true positive thresholds used for AP computation and S is the number of true positive errors. + scores: Corresponding scores for the true positives/false positives (N,). + """ + + # Ensure the number of boxes considered per class is at most `MAX_NUM_BOXES`. + if dts.shape[0] > MAX_NUM_BOXES: + dts = dts[:MAX_NUM_BOXES] + + n_threshs = len(cfg.affinity_threshs) + metrics = np.zeros((dts.shape[0], n_threshs + N_TP_ERRORS)) + + # Set the true positive metrics to np.nan since error is undefined on false positives. + metrics[:, n_threshs : n_threshs + N_TP_ERRORS] = np.nan + if gts.shape[0] == 0: + return metrics + + affinity_matrix = compute_affinity_matrix(dts, gts, cfg.affinity_fn_type) + + # Get the GT label for each max-affinity GT label, detection pair. + gt_matches = affinity_matrix.argmax(axis=1)[np.newaxis, :] + + # The affinity matrix is an N by M matrix of the detections and ground truth labels respectively. + # We want to take the corresponding affinity for each of the initial assignments using `gt_matches`. + # The following line grabs the max affinity for each detection to a ground truth label. + affinities = np.take_along_axis(affinity_matrix.T, gt_matches, axis=0).squeeze(0) + + # Find the indices of the "first" detection assigned to each GT. + unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) + for i, thresh in enumerate(cfg.affinity_threshs): + + # `tp_mask` may need to be defined differently with other affinities. + tp_mask = affinities[unique_dt_matches] > -thresh + metrics[unique_dt_matches, i] = tp_mask + + # Only compute true positive error when `thresh` is equal to the tp threshold. + is_tp_thresh = thresh == cfg.tp_thresh + # Ensure that there are true positives of the respective class in the frame. + has_true_positives = np.count_nonzero(tp_mask) > 0 + + if is_tp_thresh and has_true_positives: + dt_tp_indices = unique_dt_matches[tp_mask] + gt_tp_indices = unique_gt_matches[tp_mask] + + # Form DataFrame of shape (N, D) where D is the number of attributes in `ObjectLabelRecord`. + dt_df = pd.DataFrame([dt.__dict__ for dt in dts[dt_tp_indices]]) + gt_df = pd.DataFrame([gt.__dict__ for gt in gts[gt_tp_indices]]) + + trans_error = dist_fn(dt_df, gt_df, DistFnType.TRANSLATION) + scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) + orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) + + metrics[dt_tp_indices, n_threshs : n_threshs + N_TP_ERRORS] = np.vstack( + (trans_error, scale_error, orient_error) + ).T + return metrics + + def filter_instances( instances: List[ObjectLabelRecord], target_class_name: str, filter_metric: FilterMetric, max_detection_range: float ) -> np.ndarray: diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 3510ddb4..3fcc38ca 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -60,7 +60,7 @@ from collections import defaultdict from multiprocessing import Pool from pathlib import Path -from typing import DefaultDict, List, NamedTuple, Tuple +from typing import DefaultDict, List, NamedTuple import matplotlib import numpy as np @@ -68,18 +68,7 @@ from pandas.core import frame from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT -from argoverse.data_loading.object_label_record import read_label -from argoverse.evaluation.detection_utils import ( - AffFnType, - DistFnType, - FilterMetric, - calc_ap, - compute_affinity_matrix, - dist_fn, - filter_instances, - interp, - rank, -) +from argoverse.evaluation.detection_utils import AffFnType, DetectionCfg, FilterMetric, accumulate, calc_ap matplotlib.use("Agg") # isort:skip @@ -95,53 +84,9 @@ STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"] -MAX_SCALE_ERROR: float = 1.0 -MAX_YAW_ERROR: float = np.pi - -# Higher is better. -MIN_AP: float = 0.0 -MIN_CDS: float = 0.0 - -# Lower is better. -MAX_NORMALIZED_ATE: float = 1.0 -MAX_NORMALIZED_ASE: float = 1.0 -MAX_NORMALIZED_AOE: float = 1.0 - -# Max number of boxes considered per class per scene. -MAX_NUM_BOXES: int = 500 - SIGNIFICANT_DIGITS: float = 3 -class DetectionCfg(NamedTuple): - """Instantiates a DetectionCfg object for configuring a DetectionEvaluator. - - Args: - affinity_threshs: Affinity thresholds for determining a true positive. - affinity_fn_type: Type of affinity function to be used for calculating average precision. - n_rec_samples: Number of recall points to sample uniformly in [0, 1]. Default to 101 recall samples. - tp_thresh: Center distance threshold for the true positive metrics (in meters). - dt_classes: Detection classes for evaluation. - dt_metric: Detection metric to use for filtering of both detections and ground truth annotations. - max_dt_range: The max distance (under a specific metric in meters) for a detection or ground truth to be - considered for evaluation. - save_figs: Flag to save figures. - tp_normalization_terms: Normalization constants for ATE, ASE, and AOE. - summary_default_vals: Evaluation summary default values. - """ - - affinity_threshs: List[float] = [0.5, 1.0, 2.0, 4.0] # Meters - affinity_fn_type: AffFnType = AffFnType.CENTER - n_rec_samples: int = 101 - tp_thresh: float = 2.0 # Meters - dt_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys()) - dt_metric: FilterMetric = FilterMetric.EUCLIDEAN - max_dt_range: float = 100.0 # Meters - save_figs: bool = False - tp_normalization_terms: np.ndarray = np.array([tp_thresh, MAX_SCALE_ERROR, MAX_YAW_ERROR]) - summary_default_vals: np.ndarray = np.array([MIN_AP, tp_thresh, MAX_NORMALIZED_ASE, MAX_NORMALIZED_AOE, MIN_CDS]) - - class DetectionEvaluator(NamedTuple): """Instantiates a DetectionEvaluator object for evaluation. @@ -173,8 +118,9 @@ def evaluate(self) -> pd.DataFrame: data: DefaultDict[str, np.ndarray] = defaultdict(list) cls_to_ninst: DefaultDict[str, int] = defaultdict(int) + args = [(self.dt_root_fpath, gt_fpath, self.cfg) for gt_fpath in gt_fpaths] with Pool(os.cpu_count()) as p: - accum = p.map(self.accumulate, gt_fpaths) + accum = p.starmap(accumulate, args) for frame_stats, frame_cls_to_inst in accum: for cls_name, cls_stats in frame_stats.items(): @@ -197,113 +143,6 @@ def evaluate(self) -> pd.DataFrame: summary.loc["Average Metrics"] = summary.mean().round(SIGNIFICANT_DIGITS) return summary - def accumulate(self, gt_fpath: Path) -> Tuple[DefaultDict[str, np.ndarray], DefaultDict[str, int]]: - """Accumulate the true/false positives (boolean flags) and true positive errors for each class. - - Args: - gt_fpath: Ground truth file path. - - Returns: - cls_to_accum: Class to accumulated statistics dictionary of shape |C| -> (N, K + S) where C - is the number of detection classes, K is the number of true positive thresholds used for - AP computation, and S is the number of true positive errors. - cls_to_ninst: Mapping of shape |C| -> (1,) the class names to the number of instances in the ground - truth dataset. - """ - log_id = gt_fpath.parents[1].stem - logger.info(f"log_id = {log_id}") - ts = gt_fpath.stem.split("_")[-1] - - dt_fpath = self.dt_root_fpath / f"{log_id}/per_sweep_annotations_amodal/" f"tracked_object_labels_{ts}.json" - - dts = np.array(read_label(str(dt_fpath))) - gts = np.array(read_label(str(gt_fpath))) - - cls_to_accum = defaultdict(list) - cls_to_ninst = defaultdict(int) - for class_name in self.cfg.dt_classes: - dt_filtered = filter_instances( - dts, class_name, filter_metric=self.cfg.dt_metric, max_detection_range=self.cfg.max_dt_range - ) - gt_filtered = filter_instances( - gts, class_name, filter_metric=self.cfg.dt_metric, max_detection_range=self.cfg.max_dt_range - ) - - logger.info(f"{dt_filtered.shape[0]} detections") - logger.info(f"{gt_filtered.shape[0]} ground truth") - if dt_filtered.shape[0] > 0: - ranked_detections, scores = rank(dt_filtered) - metrics = self.assign(ranked_detections, gt_filtered) - cls_to_accum[class_name] = np.hstack((metrics, scores)) - - cls_to_ninst[class_name] = gt_filtered.shape[0] - return cls_to_accum, cls_to_ninst - - def assign(self, dts: np.ndarray, gts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - """Attempt assignment of each detection to a ground truth label. - - Args: - dts: Detections of shape (N,). - gts: Ground truth labels of shape (M,). - - Returns: - metrics: Matrix of true/false positive concatenated with true positive errors (N, K + S) where K is the number - of true positive thresholds used for AP computation and S is the number of true positive errors. - scores: Corresponding scores for the true positives/false positives (N,). - """ - - # Ensure the number of boxes considered per class is at most `MAX_NUM_BOXES`. - if dts.shape[0] > MAX_NUM_BOXES: - dts = dts[:MAX_NUM_BOXES] - - n_threshs = len(self.cfg.affinity_threshs) - metrics = np.zeros((dts.shape[0], n_threshs + N_TP_ERRORS)) - - # Set the true positive metrics to np.nan since error is undefined on false positives. - metrics[:, n_threshs : n_threshs + N_TP_ERRORS] = np.nan - if gts.shape[0] == 0: - return metrics - - affinity_matrix = compute_affinity_matrix(dts, gts, self.cfg.affinity_fn_type) - - # Get the GT label for each max-affinity GT label, detection pair. - gt_matches = affinity_matrix.argmax(axis=1)[np.newaxis, :] - - # The affinity matrix is an N by M matrix of the detections and ground truth labels respectively. - # We want to take the corresponding affinity for each of the initial assignments using `gt_matches`. - # The following line grabs the max affinity for each detection to a ground truth label. - affinities = np.take_along_axis(affinity_matrix.T, gt_matches, axis=0).squeeze(0) - - # Find the indices of the "first" detection assigned to each GT. - unique_gt_matches, unique_dt_matches = np.unique(gt_matches, return_index=True) - for i, thresh in enumerate(self.cfg.affinity_threshs): - - # `tp_mask` may need to be defined differently with other affinities. - tp_mask = affinities[unique_dt_matches] > -thresh - metrics[unique_dt_matches, i] = tp_mask - - # Only compute true positive error when `thresh` is equal to the tp threshold. - is_tp_thresh = thresh == self.cfg.tp_thresh - # Ensure that there are true positives of the respective class in the frame. - has_true_positives = np.count_nonzero(tp_mask) > 0 - - if is_tp_thresh and has_true_positives: - dt_tp_indices = unique_dt_matches[tp_mask] - gt_tp_indices = unique_gt_matches[tp_mask] - - # Form DataFrame of shape (N, D) where D is the number of attributes in `ObjectLabelRecord`. - dt_df = pd.DataFrame([dt.__dict__ for dt in dts[dt_tp_indices]]) - gt_df = pd.DataFrame([gt.__dict__ for gt in gts[gt_tp_indices]]) - - trans_error = dist_fn(dt_df, gt_df, DistFnType.TRANSLATION) - scale_error = dist_fn(dt_df, gt_df, DistFnType.SCALE) - orient_error = dist_fn(dt_df, gt_df, DistFnType.ORIENTATION) - - metrics[dt_tp_indices, n_threshs : n_threshs + N_TP_ERRORS] = np.vstack( - (trans_error, scale_error, orient_error) - ).T - return metrics - def summarize( self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int] ) -> DefaultDict[str, List]: From 69533365e2ddd695e9642aa6b2489855f622c2c8 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:19:20 -0400 Subject: [PATCH 069/113] Fixed docstring. --- argoverse/evaluation/detection_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 5128e963..5cc42f71 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -104,7 +104,9 @@ def accumulate( """Accumulate the true/false positives (boolean flags) and true positive errors for each class. Args: + dt_root_fpath: Detections root folder file path. gt_fpath: Ground truth file path. + cfg: Detection configuration. Returns: cls_to_accum: Class to accumulated statistics dictionary of shape |C| -> (N, K + S) where C From 2fdb385719fa68a74a900ab0702e7a524b4d4dca Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:20:03 -0400 Subject: [PATCH 070/113] Removed unused imports. --- argoverse/evaluation/eval_detection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 3fcc38ca..0d116cdd 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -67,8 +67,7 @@ import pandas as pd from pandas.core import frame -from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT -from argoverse.evaluation.detection_utils import AffFnType, DetectionCfg, FilterMetric, accumulate, calc_ap +from argoverse.evaluation.detection_utils import DetectionCfg, accumulate, calc_ap matplotlib.use("Agg") # isort:skip From 90363ff2d809194e0f964145c1813dfbe012c9f8 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:24:24 -0400 Subject: [PATCH 071/113] Moved additional functions out of eval_detection.py. --- argoverse/evaluation/detection_utils.py | 22 +++++++++++++++++++++ argoverse/evaluation/eval_detection.py | 26 ++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 5cc42f71..011779c7 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import DefaultDict, List, NamedTuple, Tuple +import matplotlib +import matplotlib.pyplot as plt import numpy as np import pandas as pd from scipy.spatial.distance import cdist @@ -25,6 +27,9 @@ from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label from argoverse.utils.transform import quat_argo2scipy_vectorized +matplotlib.use("Agg") + + logger = logging.getLogger(__name__) @@ -396,3 +401,20 @@ def wrap_angle(angles: np.ndarray, period: float = np.pi) -> np.ndarray: # `mods` must be nonzero, thus the image is the interval [0, π). angles[angle_complement_mask] = period - mods[angle_complement_mask] return angles + + +def plot(rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str, figs_fpath: Path) -> None: + """Plot and save the precision recall curve. + + Args: + rec_interp: Interpolated recall data of shape (N,). + prec_interp: Interpolated precision data of shape (N,). + cls_name: Class name. + figs_fpath: Path to the folder which will contain the output figures. + """ + plt.plot(rec_interp, prec_interp) + plt.title("PR Curve") + plt.xlabel("Recall") + plt.ylabel("Precision") + plt.savefig(f"{figs_fpath}/{cls_name}.png") + plt.close() diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py index 0d116cdd..cf396c41 100644 --- a/argoverse/evaluation/eval_detection.py +++ b/argoverse/evaluation/eval_detection.py @@ -62,18 +62,11 @@ from pathlib import Path from typing import DefaultDict, List, NamedTuple -import matplotlib import numpy as np import pandas as pd from pandas.core import frame -from argoverse.evaluation.detection_utils import DetectionCfg, accumulate, calc_ap - -matplotlib.use("Agg") # isort:skip - - -import matplotlib.pyplot as plt # isort:skip - +from argoverse.evaluation.detection_utils import DetectionCfg, accumulate, calc_ap, plot logger = logging.getLogger(__name__) @@ -171,7 +164,7 @@ def summarize( summary[cls_name] += [ap_th] if self.cfg.save_figs: - self.plot(recalls_interp, precisions_interp, cls_name) + plot(recalls_interp, precisions_interp, cls_name, self.figs_fpath) # AP Metric. ap = np.array(summary[cls_name][:num_ths]).mean() @@ -198,21 +191,6 @@ def summarize( logger.info(f"summary = {summary}") return summary - def plot(self, rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str) -> None: - """Plot and save the precision recall curve. - - Args: - rec_interp: Interpolated recall data of shape (N,). - prec_interp: Interpolated precision data of shape (N,). - cls_name: Class name. - """ - plt.plot(rec_interp, prec_interp) - plt.title("PR Curve") - plt.xlabel("Recall") - plt.ylabel("Precision") - plt.savefig(f"{self.figs_fpath}/{cls_name}.png") - plt.close() - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) From b7b475a2cd4d19baae0827c6d607399f86dfe0f3 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:25:23 -0400 Subject: [PATCH 072/113] Fixed plotting sorting. --- argoverse/evaluation/detection_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 011779c7..4575a7ca 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -17,7 +17,6 @@ from typing import DefaultDict, List, NamedTuple, Tuple import matplotlib -import matplotlib.pyplot as plt import numpy as np import pandas as pd from scipy.spatial.distance import cdist @@ -27,7 +26,8 @@ from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label from argoverse.utils.transform import quat_argo2scipy_vectorized -matplotlib.use("Agg") +matplotlib.use("Agg") # isort:skip +import matplotlib.pyplot as plt # isort:skip logger = logging.getLogger(__name__) From 902d3a6ffa27c8b751eb0e446bc00932d07f9f00 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 16:38:26 -0400 Subject: [PATCH 073/113] Type fix. --- argoverse/evaluation/eval_utils.py | 2 +- argoverse/utils/manhattan_search.py | 2 +- tests/test_manhattan_search.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/argoverse/evaluation/eval_utils.py b/argoverse/evaluation/eval_utils.py index c3b18b74..287187cc 100644 --- a/argoverse/evaluation/eval_utils.py +++ b/argoverse/evaluation/eval_utils.py @@ -98,7 +98,7 @@ def label_to_bbox(label: _LabelType) -> Tuple[np.ndarray, float]: p1 = np.array([+length / 2, -width / 2, -height / 2])[:, np.newaxis] p2 = np.array([-length / 2, +width / 2, -height / 2])[:, np.newaxis] - bbox = np.array([p0, p1, p2, height]) + bbox = np.array([p0, p1, p2, height], dtype=object) R = quat2rotmat((label["rotation"]["w"], label["rotation"]["x"], label["rotation"]["y"], label["rotation"]["z"])) t = np.array([label["center"]["x"], label["center"]["y"], label["center"]["z"]])[:, np.newaxis] diff --git a/argoverse/utils/manhattan_search.py b/argoverse/utils/manhattan_search.py index 44c82551..0be5eeae 100644 --- a/argoverse/utils/manhattan_search.py +++ b/argoverse/utils/manhattan_search.py @@ -146,7 +146,7 @@ def find_local_polygons( def prune_polygons_manhattan_dist( - query_pt: np.ndarray, points_xyz: np.ndarray, query_search_range_manhattan: int = 200 + query_pt: np.ndarray, points_xyz: np.ndarray, query_search_range_manhattan: float = 200.0 ) -> np.ndarray: """ Prune polygon points based on a search area defined by the manhattan distance. diff --git a/tests/test_manhattan_search.py b/tests/test_manhattan_search.py index 4732d3f6..5dc8b0c8 100644 --- a/tests/test_manhattan_search.py +++ b/tests/test_manhattan_search.py @@ -70,7 +70,7 @@ def test_find_all_polygon_bboxes_overlapping_query_bbox(polygons_and_gt_bboxes): def test_compute_polygon_bboxes(polygons_and_gt_bboxes): """Test for correctness of compute_polygon_bboxes.""" - polygon_bboxes = compute_polygon_bboxes(np.array(polygons_and_gt_bboxes[0])) + polygon_bboxes = compute_polygon_bboxes(np.array(polygons_and_gt_bboxes[0], dtype=object)) gt_polygon_bboxes = np.array(polygons_and_gt_bboxes[1]) assert np.allclose(polygon_bboxes, gt_polygon_bboxes) @@ -83,7 +83,7 @@ def test_prune_polygons_manhattan_dist_find_nearby( query_pt: np.ndarray, query_search_range_manhattan: float, gt_indices: Sequence[int], polygons_and_gt_bboxes ): """Test for correctness of prune_polygons_manhattan_dist.""" - polygons = np.array(polygons_and_gt_bboxes[0]) + polygons = np.array(polygons_and_gt_bboxes[0], dtype=object) pruned_polygons = prune_polygons_manhattan_dist(query_pt, polygons.copy(), query_search_range_manhattan) gt_pruned_polygons = np.array([polygons[i] for i in gt_indices], dtype="O") assert_np_obj_arrs_eq(gt_pruned_polygons, pruned_polygons) @@ -91,7 +91,7 @@ def test_prune_polygons_manhattan_dist_find_nearby( def test_find_local_polygons(polygons_and_gt_bboxes): """Test for correctness of find_local_polygons.""" - polygons = np.array(polygons_and_gt_bboxes[0]) + polygons = np.array(polygons_and_gt_bboxes[0], dtype=object) poly_bboxes = np.array(polygons_and_gt_bboxes[1]) query_bbox = np.array([-1.5, 0.5, 1.5, 1.5]) From 7e69da183afd2ad127ff15f9bd739b82dd60458b Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 17:43:51 -0400 Subject: [PATCH 074/113] Fixed formatting. --- demo_usage/visualize_30hz_benchmark_data_on_map.py | 6 +----- tests/test_frustum_clipping.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/demo_usage/visualize_30hz_benchmark_data_on_map.py b/demo_usage/visualize_30hz_benchmark_data_on_map.py index 01d4d92d..9100b53b 100644 --- a/demo_usage/visualize_30hz_benchmark_data_on_map.py +++ b/demo_usage/visualize_30hz_benchmark_data_on_map.py @@ -11,9 +11,6 @@ from typing import Any import imageio -# all mayavi imports MUST come before matplotlib, else Tkinter exceptions -# will be thrown, e.g. "unrecognized selector sent to instance" -import mayavi import matplotlib.pyplot as plt import numpy as np @@ -23,13 +20,12 @@ from argoverse.utils.camera_stats import RING_CAMERA_LIST, STEREO_CAMERA_LIST from argoverse.utils.cuboid_interior import filter_point_cloud_to_bbox_2D_vectorized from argoverse.utils.ffmpeg_utils import write_nonsequential_idx_video, write_video -from argoverse.utils.geometry import filter_point_cloud_to_polygon, rotate_polygon_about_pt +from argoverse.utils.geometry import rotate_polygon_about_pt from argoverse.utils.mpl_plotting_utils import draw_lane_polygons, plot_bbox_2D from argoverse.utils.pkl_utils import load_pkl_dictionary from argoverse.utils.ply_loader import load_ply from argoverse.utils.se3 import SE3 from argoverse.visualization.ground_visualization import draw_ground_pts_in_image -from argoverse.visualization.mayavi_utils import draw_lidar from argoverse.visualization.mpl_point_cloud_vis import draw_point_cloud_bev logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py index 71f22398..bdef0913 100644 --- a/tests/test_frustum_clipping.py +++ b/tests/test_frustum_clipping.py @@ -333,7 +333,7 @@ def test_generate_frustum_planes_stereo() -> None: if planes is None: assert False left_plane, right_plane, near_plane, low_plane, top_plane = planes - + fx = K[0, 0] left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0]) right_plane_gt = np.array([-fx, 0.0, img_width / 2.0, 0.0]) From ef3b3310f0a5a65fde77a53619dd217df56d50fb Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 18:31:27 -0400 Subject: [PATCH 075/113] Fixed implement errors and flake8 issues. --- argoverse/evaluation/detection_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py index 4575a7ca..f2245461 100644 --- a/argoverse/evaluation/detection_utils.py +++ b/argoverse/evaluation/detection_utils.py @@ -7,7 +7,6 @@ highest confidence prediction within a specified Euclidean distance threshold from a bird's-eye view. We prefer these metrics instead of IoU due to the increased interpretability of the error modes in a set of detections. - """ import logging @@ -27,7 +26,7 @@ from argoverse.utils.transform import quat_argo2scipy_vectorized matplotlib.use("Agg") # isort:skip -import matplotlib.pyplot as plt # isort:skip +import matplotlib.pyplot as plt # isort:skip #noqa: E402 logger = logging.getLogger(__name__) @@ -274,7 +273,7 @@ def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray: if method == InterpType.ALL: prec_interp = np.maximum.accumulate(prec[::-1])[::-1] else: - raise NotImplemented("This interpolation method is not implemented!") + raise NotImplementedError("This interpolation method is not implemented!") return prec_interp @@ -297,7 +296,7 @@ def compute_affinity_matrix( gt_centers = np.array([gt.translation for gt in gts]) sims = -cdist(dt_centers, gt_centers) else: - raise NotImplemented("This similarity metric is not implemented!") + raise NotImplementedError("This similarity metric is not implemented!") return sims @@ -358,7 +357,7 @@ def dist_fn(dts: pd.DataFrame, gts: pd.DataFrame, metric: DistFnType) -> np.ndar orientation_errors = wrap_angle(dt_yaws - gt_yaws) return orientation_errors else: - raise NotImplemented("This distance metric is not implemented!") + raise NotImplementedError("This distance metric is not implemented!") def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray: From 2e6f44dafc6e9d09e40b00ee1268622948c0cc26 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 19:27:02 -0400 Subject: [PATCH 076/113] Fixed end of line issues. --- argoverse/evaluation/competition_util.py | 2 +- argoverse/map_representation/map_viz_helper.py | 2 +- argoverse/utils/calibration.py | 2 +- argoverse/utils/make_track_label_folders.py | 3 ++- argoverse/visualization/visualization_utils.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/argoverse/evaluation/competition_util.py b/argoverse/evaluation/competition_util.py index 8ff18d53..ea64e19a 100644 --- a/argoverse/evaluation/competition_util.py +++ b/argoverse/evaluation/competition_util.py @@ -9,12 +9,12 @@ import zipfile from typing import Dict, List, Optional, Tuple, Union +import h5py import numpy as np from scipy.spatial import ConvexHull from shapely.geometry import Polygon from sklearn.cluster.dbscan_ import DBSCAN -import h5py import quaternion from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader from argoverse.data_loading.object_label_record import ObjectLabelRecord diff --git a/argoverse/map_representation/map_viz_helper.py b/argoverse/map_representation/map_viz_helper.py index ca11188a..be709806 100644 --- a/argoverse/map_representation/map_viz_helper.py +++ b/argoverse/map_representation/map_viz_helper.py @@ -8,12 +8,12 @@ import cv2 import numpy as np from colour import Color +from typing_extensions import Protocol from argoverse.utils.cv2_plotting_utils import draw_polygon_cv2, draw_polyline_cv2 from argoverse.utils.datetime_utils import generate_datetime_string from argoverse.utils.mesh_grid import get_mesh_grid_as_point_cloud from argoverse.utils.se2 import SE2 -from typing_extensions import Protocol from .lane_segment import LaneSegment diff --git a/argoverse/utils/calibration.py b/argoverse/utils/calibration.py index fd6d5fcf..bc10218c 100644 --- a/argoverse/utils/calibration.py +++ b/argoverse/utils/calibration.py @@ -8,6 +8,7 @@ import imageio import numpy as np +from typing_extensions import Literal from argoverse.data_loading.pose_loader import get_city_SE3_egovehicle_at_sensor_t from argoverse.utils.camera_stats import ( @@ -21,7 +22,6 @@ ) from argoverse.utils.se3 import SE3 from argoverse.utils.transform import quat2rotmat -from typing_extensions import Literal logger = logging.getLogger(__name__) diff --git a/argoverse/utils/make_track_label_folders.py b/argoverse/utils/make_track_label_folders.py index 976f5674..bfc216b5 100644 --- a/argoverse/utils/make_track_label_folders.py +++ b/argoverse/utils/make_track_label_folders.py @@ -4,9 +4,10 @@ import sys from typing import Any, Dict, List -from argoverse.utils.json_utils import read_json_file, save_json_dict from typing_extensions import TypedDict +from argoverse.utils.json_utils import read_json_file, save_json_dict + root_dir = sys.argv[1] print("root dir = ", root_dir) diff --git a/argoverse/visualization/visualization_utils.py b/argoverse/visualization/visualization_utils.py index cdd60e0b..81948146 100644 --- a/argoverse/visualization/visualization_utils.py +++ b/argoverse/visualization/visualization_utils.py @@ -7,13 +7,13 @@ import cv2 import matplotlib.pyplot as plt import numpy as np +from PIL import Image from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.utils.calibration import Calibration, determine_valid_cam_coords, proj_cam_to_uv from argoverse.utils.frustum_clipping import generate_frustum_planes -from PIL import Image point_size = 0.01 axes_limits = [[-10, 10], [-10, 10], [-3, 10]] # X axis range # Y axis range # Z axis range From f6902244c4c05d6744c8fe3dabd2d9cfebcb8400 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 20:52:57 -0400 Subject: [PATCH 077/113] Updated isort to align with black + tons of typing fixes. --- .pre-commit-config.yaml | 4 ++-- argoverse/evaluation/competition_util.py | 2 +- argoverse/utils/bfs.py | 12 ++++++------ argoverse/utils/ffmpeg_utils.py | 1 - argoverse/utils/make_att_files.py | 6 +++--- .../test_plane_visualization_utils.py | 3 ++- sphinx/conf.py | 14 ++++++++------ tests/test_bfs.py | 9 +++------ tests/test_cuboid_interior.py | 1 - tests/test_geometry.py | 1 - 10 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec6090be..31f0b615 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,10 +10,10 @@ repos: - id: end-of-file-fixer - id: debug-statements - repo: https://github.com/pre-commit/mirrors-isort.git - rev: dc9ed97 + rev: 687aecc hooks: - id: isort - args: ["-w", "120", "--multi-line=3", "--trailing-comma", "--force-grid-wrap=0", "--use-parentheses"] + args: ["-w", "120", "--profile", "black"] - repo: https://github.com/python/black.git rev: 1bbb01b hooks: diff --git a/argoverse/evaluation/competition_util.py b/argoverse/evaluation/competition_util.py index ea64e19a..c9095289 100644 --- a/argoverse/evaluation/competition_util.py +++ b/argoverse/evaluation/competition_util.py @@ -11,11 +11,11 @@ import h5py import numpy as np +import quaternion from scipy.spatial import ConvexHull from shapely.geometry import Polygon from sklearn.cluster.dbscan_ import DBSCAN -import quaternion from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader from argoverse.data_loading.object_label_record import ObjectLabelRecord from argoverse.utils.se3 import SE3 diff --git a/argoverse/utils/bfs.py b/argoverse/utils/bfs.py index 02ee3fd4..9e171607 100644 --- a/argoverse/utils/bfs.py +++ b/argoverse/utils/bfs.py @@ -5,7 +5,7 @@ from typing import Any, Collection, List, Mapping, MutableMapping, MutableSequence, Sequence -def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_depth: int = 4) -> Sequence[Sequence[str]]: +def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_depth: int = 4) -> List[List[str]]: """Run Breadth-First-Search. Cycles are allowed and are accounted for. Find (u,v) edges in E of graph (V,E) @@ -27,14 +27,14 @@ def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_dept dists[v] = float("inf") dists[start] = 0 - paths: MutableSequence[MutableSequence[str]] = [] + paths: List[List[str]] = [] # maintain a queue of paths - queue: MutableSequence[MutableSequence[str]] = [] + queue: List[List[str]] = [] # push the first path into the queue queue.append([start]) while queue: # len(q) > 0: # get the first path from the queue - path: MutableSequence[str] = queue.pop(0) + path: List[str] = queue.pop(0) # get the last node from the path u: str = path[-1] # max depth already exceeded, terminate @@ -43,7 +43,7 @@ def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_dept # enumerate all adjacent nodes, construct a new path and push it into the queue for v in graph.get(u, []): if dists[v] == float("inf"): - new_path: MutableSequence[str] = list(path) + new_path: List[str] = list(path) new_path.append(v) queue.append(new_path) dists[v] = dists[u] + 1 @@ -52,7 +52,7 @@ def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_dept return remove_duplicate_paths(paths) -def remove_duplicate_paths(paths: Sequence[Sequence[Any]]) -> Sequence[Sequence[str]]: +def remove_duplicate_paths(paths: List[List[Any]]) -> List[List[str]]: """Remove duplicate subpaths from a set of paths. For example, if ``['1', '2', '6']`` and ``['1', '2']`` are diff --git a/argoverse/utils/ffmpeg_utils.py b/argoverse/utils/ffmpeg_utils.py index a0f99ef1..89a21646 100644 --- a/argoverse/utils/ffmpeg_utils.py +++ b/argoverse/utils/ffmpeg_utils.py @@ -3,7 +3,6 @@ from argoverse.utils.subprocess_utils import run_command - """ Create a high-quality video using the encoder x264. For x264, the valid Constant Rate Factor (crf) range is 0-51. diff --git a/argoverse/utils/make_att_files.py b/argoverse/utils/make_att_files.py index 6c8c0bfd..7f3d32be 100644 --- a/argoverse/utils/make_att_files.py +++ b/argoverse/utils/make_att_files.py @@ -6,12 +6,12 @@ import cv2 import matplotlib.pyplot as plt import numpy as np -import scipy.interpolate as interpolate - -import argoverse import open3d as o3d +import scipy.interpolate as interpolate import torch import torch.nn.functional as F + +import argoverse from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader from argoverse.utils.json_utils import read_json_file from argoverse.utils.pkl_utils import save_pkl_dictionary diff --git a/integration_tests/test_plane_visualization_utils.py b/integration_tests/test_plane_visualization_utils.py index abfea240..4be8292b 100644 --- a/integration_tests/test_plane_visualization_utils.py +++ b/integration_tests/test_plane_visualization_utils.py @@ -9,11 +9,12 @@ try: import mayavi.mlab + from argoverse.utils.frustum_clipping import generate_frustum_planes from argoverse.utils.plane_visualization_utils import ( - populate_frustum_voxels, get_perpendicular, plot_frustum_planes_and_normals, + populate_frustum_voxels, ) except ImportError: MAYAVI_MISSING = True diff --git a/sphinx/conf.py b/sphinx/conf.py index ccc0b54e..4bd4cbce 100644 --- a/sphinx/conf.py +++ b/sphinx/conf.py @@ -3,10 +3,12 @@ import re import shutil from pathlib import Path +from typing import Dict, List -import sphinx.ext.apidoc from recommonmark.transform import AutoStructify +import sphinx.ext.apidoc + _BADGE_REGEX = re.compile(r"^(?:\[!\[[^[]*\](\([^)]+\)\]\([^)]+\)))", re.M) _TOC_REGEX = re.compile(r"(## Table of Contents.+- \[Installation\][^\n]*\n)", re.S) _NOTEBOOK_REGEX = re.compile(r"(?:./)?(demo_usage/.+.ipynb)", re.M) @@ -39,13 +41,13 @@ "sphinx_autodoc_typehints", "recommonmark", ] -templates_path = [] -exclude_patterns = [] -source_suffix = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"} +templates_path: List = [] +exclude_patterns: List = [] +source_suffix: Dict = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"} # -- Options for HTML output ------------------------------------------------- -html_theme = "sphinx_rtd_theme" -html_static_path = [] +html_theme: str = "sphinx_rtd_theme" +html_static_path: List = [] # -- Extension configuration ------------------------------------------------- diff --git a/tests/test_bfs.py b/tests/test_bfs.py index d8e124b0..3bcd224b 100644 --- a/tests/test_bfs.py +++ b/tests/test_bfs.py @@ -2,20 +2,17 @@ # -from typing import Mapping, MutableSequence, Sequence +from typing import List, Mapping, MutableSequence, Sequence from argoverse.utils.bfs import bfs_enumerate_paths - """ Collection of unit tests to verify that Breadth-First search utility on semantic lane graph works properly. """ -def compare_paths( - paths_lhs: MutableSequence[MutableSequence[str]], paths_rhs: MutableSequence[MutableSequence[str]] -) -> bool: +def compare_paths(paths_lhs: List[List[str]], paths_rhs: List[List[str]]) -> bool: """ Compare two input paths for equality. @@ -48,7 +45,7 @@ def get_sample_graph() -> Mapping[str, Sequence[str]]: def test_bfs_enumerate_paths_depth3() -> None: """Graph is in adjacent list representation.""" graph = get_sample_graph() - paths_ref_depth3 = [ + paths_ref_depth3: List[List[str]] = [ ["1", "3"], ["1", "2", "6"], ["1", "4", "8"], diff --git a/tests/test_cuboid_interior.py b/tests/test_cuboid_interior.py index 6c52d648..d54c2cb3 100644 --- a/tests/test_cuboid_interior.py +++ b/tests/test_cuboid_interior.py @@ -9,7 +9,6 @@ filter_point_cloud_to_bbox_3D_vectorized, ) - """ Run it with "pytest tracker_tools_tests.py" """ diff --git a/tests/test_geometry.py b/tests/test_geometry.py index 66eed30d..ecad3957 100644 --- a/tests/test_geometry.py +++ b/tests/test_geometry.py @@ -4,7 +4,6 @@ from argoverse.utils.geometry import filter_point_cloud_to_polygon, point_inside_polygon, rotate_polygon_about_pt - """ Unit tests for argoverse/utils/geometry.py """ From aab2582b68961cb225f2b469eb4371f80ee9ab3b Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Wed, 30 Sep 2020 21:50:24 -0400 Subject: [PATCH 078/113] Removed legacy typing. --- argoverse/utils/bfs.py | 2 +- docs/_modules/argoverse/utils/bfs.html | 10 +++++----- docs/_modules/typing.html | 8 ++++---- tests/test_bfs.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/argoverse/utils/bfs.py b/argoverse/utils/bfs.py index 9e171607..455a4983 100644 --- a/argoverse/utils/bfs.py +++ b/argoverse/utils/bfs.py @@ -2,7 +2,7 @@ """Breadth-First-Search of Graphs.""" -from typing import Any, Collection, List, Mapping, MutableMapping, MutableSequence, Sequence +from typing import Any, Collection, List, Mapping, MutableMapping, Sequence def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_depth: int = 4) -> List[List[str]]: diff --git a/docs/_modules/argoverse/utils/bfs.html b/docs/_modules/argoverse/utils/bfs.html index 80f81214..1dbc60f5 100644 --- a/docs/_modules/argoverse/utils/bfs.html +++ b/docs/_modules/argoverse/utils/bfs.html @@ -146,7 +146,7 @@

Source code for argoverse.utils.bfs

 
 """Breadth-First-Search of Graphs."""
 
-from typing import Any, Collection, List, Mapping, MutableMapping, MutableSequence, Sequence
+from typing import Any, Collection, List, Mapping, MutableMapping, List, Sequence
 
 
 
[docs]def bfs_enumerate_paths(graph: Mapping[str, Sequence[str]], start: str, max_depth: int = 4) -> Sequence[Sequence[str]]:
@@ -171,14 +171,14 @@ 
Source code for argoverse.utils.bfs
             dists[v] = float("inf")
 
     dists[start] = 0
-    paths: MutableSequence[MutableSequence[str]] = []
+    paths: List[List[str]] = []
     # maintain a queue of paths
-    queue: MutableSequence[MutableSequence[str]] = []
+    queue: List[List[str]] = []
     # push the first path into the queue
     queue.append([start])
     while queue:  # len(q) > 0:
         # get the first path from the queue
-        path: MutableSequence[str] = queue.pop(0)
+        path: List[str] = queue.pop(0)
         # get the last node from the path
         u: str = path[-1]
         # max depth already exceeded, terminate
@@ -187,7 +187,7 @@ 
Source code for argoverse.utils.bfs
         # enumerate all adjacent nodes, construct a new path and push it into the queue
         for v in graph.get(u, []):
             if dists[v] == float("inf"):
-                new_path: MutableSequence[str] = list(path)
+                new_path: List[str] = list(path)
                 new_path.append(v)
                 queue.append(new_path)
                 dists[v] = dists[u] + 1
diff --git a/docs/_modules/typing.html b/docs/_modules/typing.html
index 5f5657c6..9cda8e09 100644
--- a/docs/_modules/typing.html
+++ b/docs/_modules/typing.html
@@ -192,7 +192,7 @@ 
Source code for typing
     'Mapping',
     'MappingView',
     'MutableMapping',
-    'MutableSequence',
+    'List',
     'MutableSet',
     'Sequence',
     'Sized',
@@ -2015,7 +2015,7 @@ 
Source code for typing
         __slots__ = ()
 
 
-class MutableSequence(Sequence[T], extra=collections_abc.MutableSequence):
+class List(Sequence[T], extra=collections_abc.List):
     __slots__ = ()
 
 
@@ -2023,7 +2023,7 @@ 
Source code for typing
     __slots__ = ()
 
 
-class List(list, MutableSequence[T], extra=list):
+class List(list, List[T], extra=list):
 
     __slots__ = ()
 
@@ -2034,7 +2034,7 @@ 
Source code for typing
         return _generic_new(list, cls, *args, **kwds)
 
 
-class Deque(collections.deque, MutableSequence[T], extra=collections.deque):
+class Deque(collections.deque, List[T], extra=collections.deque):
 
     __slots__ = ()
 
diff --git a/tests/test_bfs.py b/tests/test_bfs.py
index 3bcd224b..00b11c8f 100644
--- a/tests/test_bfs.py
+++ b/tests/test_bfs.py
@@ -2,7 +2,7 @@
 
 # 
 
-from typing import List, Mapping, MutableSequence, Sequence
+from typing import List, Mapping, Sequence
 
 from argoverse.utils.bfs import bfs_enumerate_paths
 

From 78a6e53e8c0c40838b7f99f8bf4fd2a2cacaed58 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 30 Sep 2020 22:20:12 -0400
Subject: [PATCH 079/113] Fix end-of-file error.

---
 tests/test_data/json_save_test_file.json | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tests/test_data/json_save_test_file.json

diff --git a/tests/test_data/json_save_test_file.json b/tests/test_data/json_save_test_file.json
new file mode 100644
index 00000000..0315c574
--- /dev/null
+++ b/tests/test_data/json_save_test_file.json
@@ -0,0 +1 @@
+{"a": 1, "b": null, "c": 9999, "d": "abc"}

From 84360d7a73ecee5a721c2e6861bc80b913d80654 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 30 Sep 2020 22:50:01 -0400
Subject: [PATCH 080/113] Increase travis timeout.

---
 .travis.yml                             | 2 +-
 argoverse/evaluation/detection_utils.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1171a969..99a21691 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ install:
     - pip install tox-travis==0.12 tox==3.12.1 pre-commit==1.17.0
 
 script:
-    - travis_wait 30 tox
+    - travis_wait 45 tox
     - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then pre-commit run --show-diff-on-failure --origin $TRAVIS_COMMIT --source $TRAVIS_BRANCH; fi'
 
 notifications:
diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index f2245461..ac729710 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -26,7 +26,7 @@
 from argoverse.utils.transform import quat_argo2scipy_vectorized
 
 matplotlib.use("Agg")  # isort:skip
-import matplotlib.pyplot as plt  # isort:skip  #noqa: E402
+import matplotlib.pyplot as plt  # isort:skip  # noqa: E402
 
 
 logger = logging.getLogger(__name__)

From bf099e44f1a31a56670ad1a414dd2a9631b8229d Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 30 Sep 2020 23:29:13 -0400
Subject: [PATCH 081/113] mypy fixes.

---
 .pre-commit-config.yaml                 | 3 ++-
 argoverse/evaluation/detection_utils.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 31f0b615..5084dc03 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -22,5 +22,6 @@ repos:
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: d6e31ae
     hooks:
-    -   id: mypy
+    - id: mypy
+      args: ["--ignore-missing", "--strict"]
 exclude: "docs/.*$"
diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index ac729710..4a3942a2 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -149,7 +149,7 @@ def accumulate(
     return cls_to_accum, cls_to_ninst
 
 
-def assign(dts: np.ndarray, gts: np.ndarray, cfg: DetectionCfg) -> Tuple[np.ndarray, np.ndarray]:
+def assign(dts: np.ndarray, gts: np.ndarray, cfg: DetectionCfg) -> np.ndarray:
     """Attempt assignment of each detection to a ground truth label.
 
     Args:

From 4da6b956233d6256414adb671a9eda7e1651ea9d Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 30 Sep 2020 23:46:36 -0400
Subject: [PATCH 082/113] mypy fixes.

---
 tests/test_cuboid_interior.py    | 18 +++++++++++-------
 tests/test_interpolate.py        | 28 ++++++++++++++--------------
 tests/test_mesh_grid.py          |  8 ++++----
 tests/test_mpl_plotting_utils.py | 10 +++++-----
 tests/test_se2.py                | 14 +++++++-------
 5 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/tests/test_cuboid_interior.py b/tests/test_cuboid_interior.py
index d54c2cb3..076dab3b 100644
--- a/tests/test_cuboid_interior.py
+++ b/tests/test_cuboid_interior.py
@@ -1,5 +1,7 @@
 # 
 
+from typing import Tuple
+
 import numpy as np
 
 from argoverse.utils.cuboid_interior import (
@@ -14,7 +16,7 @@
 """
 
 
-def get_scenario_1():
+def get_scenario_1() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
     """
     Form an arbitrary 3d cuboid, and call it "Custom Cuboid 1".
 
@@ -78,7 +80,7 @@ def get_scenario_1():
     return pc_raw, bbox_3d, gt_segment, gt_is_valid
 
 
-def test_extract_pc_in_box3d_hull():
+def test_extract_pc_in_box3d_hull() -> None:
     """
     """
     pc_raw, bbox_3d, gt_segment, gt_is_valid = get_scenario_1()
@@ -88,7 +90,7 @@ def test_extract_pc_in_box3d_hull():
     assert np.array_equal(is_valid, gt_is_valid)
 
 
-def test_3d_cuboid_interior_test1():
+def test_3d_cuboid_interior_test1() -> None:
     """
         Generate 6 points just outside the volume of a 3d cuboid, and
         6 points just inside the volume. Verify that we can isolate the
@@ -115,14 +117,12 @@ def test_3d_cuboid_interior_test1():
     assert np.array_equal(gt_is_valid, is_valid)
 
 
-def test_2d_cuboid_interior_test1():
+def test_2d_cuboid_interior_test1() -> None:
     """Test 2d version of the 3d projection above.  This is a similar
     test except it slices away the z dimension.
 
     """
-    pc_raw, bbox_3d, gt_segment, gt_is_valid = get_scenario_1()
-
-    to_2d = lambda array: array[:, :2]
+    pc_raw, bbox_3d, gt_segment, _ = get_scenario_1()
 
     pc_raw_2d = np.unique(to_2d(pc_raw), axis=0)
     bbox_2d = to_2d(bbox_3d)[(0, 1, 4, 5), :]
@@ -135,3 +135,7 @@ def test_2d_cuboid_interior_test1():
     filtered_points = filter_point_cloud_to_bbox(bbox_2d, pc_raw_2d)
 
     assert np.array_equal(filtered_points, gt_segment_2d)
+
+
+def to_2d(coords: np.ndarray) -> np.ndarray:
+    return coords[:, :2]
diff --git a/tests/test_interpolate.py b/tests/test_interpolate.py
index f4d20361..e873ef0c 100644
--- a/tests/test_interpolate.py
+++ b/tests/test_interpolate.py
@@ -7,7 +7,7 @@
 from argoverse.utils.interpolate import compute_lane_width, compute_mid_pivot_arc, compute_midpoint_line, interp_arc
 
 
-def test_compute_lane_width_straight():
+def test_compute_lane_width_straight() -> None:
     """
         Compute the lane width of the following straight lane segment
         (waypoints indicated with "o" symbol):
@@ -31,7 +31,7 @@ def test_compute_lane_width_straight():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_lane_width_telescoping():
+def test_compute_lane_width_telescoping() -> None:
     """
         Compute the lane width of the following straight lane segment
         (waypoints indicated with "o" symbol):
@@ -57,7 +57,7 @@ def test_compute_lane_width_telescoping():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_lane_width_curved_width1():
+def test_compute_lane_width_curved_width1() -> None:
     """
         Compute the lane width of the following curved lane segment
         Should have width 1 at each pair of boundary waypoints.
@@ -82,7 +82,7 @@ def test_compute_lane_width_curved_width1():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_lane_width_curved_not_width1():
+def test_compute_lane_width_curved_not_width1() -> None:
     """
         Compute the lane width of the following curved lane segment
 
@@ -109,7 +109,7 @@ def test_compute_lane_width_curved_not_width1():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_mid_pivot_arc_3pt_cul_de_sac():
+def test_compute_mid_pivot_arc_3pt_cul_de_sac() -> None:
     """
     Make sure we handle the cul-de-sac case correctly.
 
@@ -144,7 +144,7 @@ def test_compute_mid_pivot_arc_3pt_cul_de_sac():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_mid_pivot_arc_5pt_cul_de_sac():
+def test_compute_mid_pivot_arc_5pt_cul_de_sac() -> None:
     """
     Make sure we handle the cul-de-sac case correctly.
 
@@ -179,7 +179,7 @@ def test_compute_mid_pivot_arc_5pt_cul_de_sac():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_cul_de_sac_right_onept():
+def test_compute_midpoint_line_cul_de_sac_right_onept() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -196,7 +196,7 @@ def test_compute_midpoint_line_cul_de_sac_right_onept():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_cul_de_sac_left_onept():
+def test_compute_midpoint_line_cul_de_sac_left_onept() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -213,7 +213,7 @@ def test_compute_midpoint_line_cul_de_sac_left_onept():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_straightline_maintain_5_waypts():
+def test_compute_midpoint_line_straightline_maintain_5_waypts() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -229,7 +229,7 @@ def test_compute_midpoint_line_straightline_maintain_5_waypts():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_straightline_maintain_4_waypts():
+def test_compute_midpoint_line_straightline_maintain_4_waypts() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -245,7 +245,7 @@ def test_compute_midpoint_line_straightline_maintain_4_waypts():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_straightline_maintain_3_waypts():
+def test_compute_midpoint_line_straightline_maintain_3_waypts() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -261,7 +261,7 @@ def test_compute_midpoint_line_straightline_maintain_3_waypts():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_straightline_maintain_2_waypts():
+def test_compute_midpoint_line_straightline_maintain_2_waypts() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -277,7 +277,7 @@ def test_compute_midpoint_line_straightline_maintain_2_waypts():
     assert np.isclose(lane_width, gt_lane_width)
 
 
-def test_compute_midpoint_line_curved_maintain_4_waypts():
+def test_compute_midpoint_line_curved_maintain_4_waypts() -> None:
     """
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
@@ -306,7 +306,7 @@ def test_compute_midpoint_line_curved_maintain_4_waypts():
     assert np.allclose(centerline_pts[-1], gt_centerline_pts[-1])
 
 
-def test_interp_arc_straight_line():
+def test_interp_arc_straight_line() -> None:
     """ """
     pts = np.array([[-10, 0], [10, 0]])
     interp_pts = interp_arc(t=3, px=pts[:, 0], py=pts[:, 1])
diff --git a/tests/test_mesh_grid.py b/tests/test_mesh_grid.py
index c5b3a2b1..dfc540c9 100644
--- a/tests/test_mesh_grid.py
+++ b/tests/test_mesh_grid.py
@@ -3,7 +3,7 @@
 from argoverse.utils.mesh_grid import get_mesh_grid_as_point_cloud
 
 
-def test_get_mesh_grid_as_point_cloud_3x3square():
+def test_get_mesh_grid_as_point_cloud_3x3square() -> None:
     """
     Sample a regular grid and return the (x,y) coordinates
     of the sampled points.
@@ -35,7 +35,7 @@ def test_get_mesh_grid_as_point_cloud_3x3square():
     assert np.allclose(gt_pts, pts)
 
 
-def test_get_mesh_grid_as_point_cloud_3x2rect():
+def test_get_mesh_grid_as_point_cloud_3x2rect() -> None:
     """
     Sample a regular grid and return the (x,y) coordinates
     of the sampled points.
@@ -54,7 +54,7 @@ def test_get_mesh_grid_as_point_cloud_3x2rect():
     assert np.allclose(gt_pts, pts)
 
 
-def test_get_mesh_grid_as_point_cloud_single_pt():
+def test_get_mesh_grid_as_point_cloud_single_pt() -> None:
     """
     Sample a regular grid and return the (x,y) coordinates
     of the sampled points.
@@ -73,7 +73,7 @@ def test_get_mesh_grid_as_point_cloud_single_pt():
     assert np.allclose(gt_pts, pts)
 
 
-def test_get_mesh_grid_as_point_cloud_downsample():
+def test_get_mesh_grid_as_point_cloud_downsample() -> None:
     """
     Sample a regular grid and return the (x,y) coordinates
     of the sampled points.
diff --git a/tests/test_mpl_plotting_utils.py b/tests/test_mpl_plotting_utils.py
index 796fca3c..083484d0 100644
--- a/tests/test_mpl_plotting_utils.py
+++ b/tests/test_mpl_plotting_utils.py
@@ -15,7 +15,7 @@
 )
 
 
-def test_draw_polygon_mpl_smokescreen_nolinewidth():
+def test_draw_polygon_mpl_smokescreen_nolinewidth() -> None:
     """
         """
     ax = plt.axes([1, 1, 1, 1])
@@ -27,7 +27,7 @@ def test_draw_polygon_mpl_smokescreen_nolinewidth():
     plt.close("all")
 
 
-def test_draw_polygon_mpl_smokescreen_with_linewidth():
+def test_draw_polygon_mpl_smokescreen_with_linewidth() -> None:
     """
         """
     ax = plt.axes([1, 1, 1, 1])
@@ -40,7 +40,7 @@ def test_draw_polygon_mpl_smokescreen_with_linewidth():
     plt.close("all")
 
 
-def test_plot_lane_segment_patch_smokescreen():
+def test_plot_lane_segment_patch_smokescreen() -> None:
     """
         """
     ax = plt.axes([1, 1, 1, 1])
@@ -51,7 +51,7 @@ def test_plot_lane_segment_patch_smokescreen():
     plt.close("all")
 
 
-def test_plot_nearby_centerlines_smokescreen():
+def test_plot_nearby_centerlines_smokescreen() -> None:
     """
         """
     ax = plt.axes([1, 1, 1, 1])
@@ -72,7 +72,7 @@ def test_plot_nearby_centerlines_smokescreen():
     plt.close("all")
 
 
-def test_animate_polyline_smokescreen():
+def test_animate_polyline_smokescreen() -> None:
     """
         
         """
diff --git a/tests/test_se2.py b/tests/test_se2.py
index 2518e377..f95ac25d 100644
--- a/tests/test_se2.py
+++ b/tests/test_se2.py
@@ -22,7 +22,7 @@ def rotation_matrix_from_rotation(theta: float) -> np.ndarray:
     return np.array([[cos_theta, -sin_theta], [sin_theta, cos_theta]])
 
 
-def test_SE2_constructor():
+def test_SE2_constructor() -> None:
     """Test for construction of an arbitrary SE2."""
     theta = 2 * np.pi / 7.0
     rotation_matrix = rotation_matrix_from_rotation(theta)
@@ -46,7 +46,7 @@ def test_SE2_constructor():
         SE2(rotation_matrix, np.array([1, 2, 3]))
 
 
-def test_SE2_transform_point_cloud_identity():
+def test_SE2_transform_point_cloud_identity() -> None:
     """Test that transformation by an Identity SE2 does not change pointclouds."""
     pts = np.array([[0.5, 0], [1, -0.5], [1.5, 0], [2, -1]])
     dst_se2_src = SE2(rotation=np.eye(2), translation=np.zeros(2))
@@ -61,7 +61,7 @@ def test_SE2_transform_point_cloud_identity():
         dst_se2_src.transform_point_cloud(np.random.rand(1, 3))
 
 
-def test_SE2_transform_point_cloud_pi_radians():
+def test_SE2_transform_point_cloud_pi_radians() -> None:
     """Test for validity of results of transformation."""
     pts = np.array([[0.5, 0], [1, -0.5], [1.5, 0], [2, -1]])
     theta = np.pi
@@ -75,7 +75,7 @@ def test_SE2_transform_point_cloud_pi_radians():
     assert np.allclose(transformed_pts, gt_transformed_pts)
 
 
-def test_SE2_inverse_transform_point_cloud_identity():
+def test_SE2_inverse_transform_point_cloud_identity() -> None:
     """Test that inverse transforming by Identity does not affect the pointclouds."""
     transformed_pts = np.array([[0.5, 0], [1, -0.5], [1.5, 0], [2, -1]])
     dst_se2_src = SE2(rotation=np.eye(2), translation=np.zeros(2))
@@ -86,7 +86,7 @@ def test_SE2_inverse_transform_point_cloud_identity():
         dst_se2_src.transform_point_cloud(np.random.rand(1, 3))
 
 
-def test_SE2_inverse_transform_point_cloud_pi_radians():
+def test_SE2_inverse_transform_point_cloud_pi_radians() -> None:
     """Test for validity of inverse transformation by an SE2."""
     transformed_pts = np.array([[1.5, 2.0], [1, 2.5], [0.5, 2], [0, 3.0]])
     theta = np.pi
@@ -99,7 +99,7 @@ def test_SE2_inverse_transform_point_cloud_pi_radians():
     assert np.allclose(pts, gt_pts)
 
 
-def test_SE2_chaining_transforms():
+def test_SE2_chaining_transforms() -> None:
     """Test for correctness of SE2 chaining / composing."""
     theta = np.pi
     rotation_matrix = rotation_matrix_from_rotation(theta)
@@ -115,7 +115,7 @@ def test_SE2_chaining_transforms():
     assert np.allclose(fr2_se2_fr0.transform_matrix, np.eye(3))
 
 
-def test_SE2_inverse():
+def test_SE2_inverse() -> None:
     """Test for numerical correctess of the inverse functionality."""
     src_pts_gt = np.array([[1, 0], [2, 0]])
 

From e5a50a51bd8cf842e125a18d5549835302e5f962 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 30 Sep 2020 23:55:16 -0400
Subject: [PATCH 083/113] mypy fixes.

---
 argoverse/evaluation/eval_detection.py | 4 ++--
 argoverse/evaluation/eval_tracking.py  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index cf396c41..80e33dbe 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -137,7 +137,7 @@ def evaluate(self) -> pd.DataFrame:
 
     def summarize(
         self, data: DefaultDict[str, np.ndarray], cls_to_ninst: DefaultDict[str, int]
-    ) -> DefaultDict[str, List]:
+    ) -> DefaultDict[str, List[float]]:
         """Calculate and print the detection metrics.
 
         Args:
@@ -147,7 +147,7 @@ def summarize(
         Returns:
             summary: The summary statistics.
         """
-        summary: DefaultDict[str, List] = defaultdict(list)
+        summary: DefaultDict[str, List[float]] = defaultdict(list)
         recalls_interp = np.linspace(0, 1, self.cfg.n_rec_samples)
         num_ths = len(self.cfg.affinity_threshs)
         if not self.figs_fpath.is_dir():
diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py
index fd6a0505..10ab960e 100644
--- a/argoverse/evaluation/eval_tracking.py
+++ b/argoverse/evaluation/eval_tracking.py
@@ -100,9 +100,9 @@ def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str
         dist = wrap_angle(theta).item()
 
         # Convert to degrees.
-        return np.rad2deg(dist)
+        return float(np.rad2deg(dist))
     else:
-        raise ValueError("Not implemented..")
+        raise NotImplementedError("Not implemented..")
 
 
 def eval_tracks(

From 589e89780557b060f5ae73200951631938c45f57 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 08:08:55 -0400
Subject: [PATCH 084/113] Typing fixes.

---
 tests/test_geometry.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tests/test_geometry.py b/tests/test_geometry.py
index ecad3957..a6f29d45 100644
--- a/tests/test_geometry.py
+++ b/tests/test_geometry.py
@@ -9,7 +9,7 @@
 """
 
 
-def test_rotate_polygon_about_pt_2d_triangle_0deg_origin():
+def test_rotate_polygon_about_pt_2d_triangle_0deg_origin() -> None:
     """
         Rotate a triangle 0 degrees counterclockwise, about the origin
         """
@@ -23,7 +23,7 @@ def test_rotate_polygon_about_pt_2d_triangle_0deg_origin():
     assert np.allclose(polygon_pts, rot_polygon)
 
 
-def test_rotate_polygon_about_pt_2d_triangle_90deg_origin():
+def test_rotate_polygon_about_pt_2d_triangle_90deg_origin() -> None:
     """
         Rotate a triangle 90 degrees counterclockwise, about the origin
         """
@@ -39,7 +39,7 @@ def test_rotate_polygon_about_pt_2d_triangle_90deg_origin():
     assert np.allclose(gt_rot_polygon, rot_polygon)
 
 
-def test_rotate_polygon_about_pt_2d_triangle_0deg_nonorigin():
+def test_rotate_polygon_about_pt_2d_triangle_0deg_nonorigin() -> None:
     """
         Rotate a triangle 0 degrees counterclockwise, but this time
         not rotating about the origin.
@@ -54,7 +54,7 @@ def test_rotate_polygon_about_pt_2d_triangle_0deg_nonorigin():
     assert np.allclose(polygon_pts, rot_polygon)
 
 
-def test_rotate_polygon_about_pt_2d_triangle_90deg_nonorigin():
+def test_rotate_polygon_about_pt_2d_triangle_90deg_nonorigin() -> None:
     """
         Rotate a triangle 90 degrees counterclockwise, but this time
         not rotating about the origin. Instead we rotate about (2,2).
@@ -71,7 +71,7 @@ def test_rotate_polygon_about_pt_2d_triangle_90deg_nonorigin():
     assert np.allclose(gt_rot_polygon, rot_polygon)
 
 
-def test_rotate_polygon_about_pt_3d():
+def test_rotate_polygon_about_pt_3d() -> None:
     """
     Rotate a point cloud in xy plane, but keep z fixed. in other words,
     perform a 3D rotation about Z-axis (rotation about yaw axis).
@@ -93,7 +93,7 @@ def test_rotate_polygon_about_pt_3d():
     assert np.allclose(rotated_pts, gt_rotated_pts)
 
 
-def test_filter_point_cloud_to_polygon_2d_triangle():
+def test_filter_point_cloud_to_polygon_2d_triangle() -> None:
     """
         Test points that fall within a triangle symbol centered at
         the origin. The shape resembles:
@@ -130,7 +130,7 @@ def test_filter_point_cloud_to_polygon_2d_triangle():
     assert np.allclose(point_cloud_2d[interior_gt_bool], interior_pts)
 
 
-def test_filter_point_cloud_to_polygon_2d_triangle_and_3d_pointcloud():
+def test_filter_point_cloud_to_polygon_2d_triangle_and_3d_pointcloud() -> None:
     """
     Test points that fall within a triangle symbol centered at
     the origin. The shape resembles:
@@ -168,7 +168,7 @@ def test_filter_point_cloud_to_polygon_2d_triangle_and_3d_pointcloud():
     assert np.allclose(point_cloud_2d[interior_gt_bool], interior_pts)
 
 
-def test_filter_point_cloud_to_polygon_2d_triangle_all_outside():
+def test_filter_point_cloud_to_polygon_2d_triangle_all_outside() -> None:
     """
     Test points that fall within a triangle symbol centered at
     the origin. The shape resembles:
@@ -200,7 +200,7 @@ def test_filter_point_cloud_to_polygon_2d_triangle_all_outside():
     assert interior_pts is None
 
 
-def test_filter_point_cloud_to_polygon_2d_redcross():
+def test_filter_point_cloud_to_polygon_2d_redcross() -> None:
     """
     Test points that fall within a red cross symbol centered at
     the origin.
@@ -225,7 +225,9 @@ def test_filter_point_cloud_to_polygon_2d_redcross():
     assert np.allclose(point_cloud_2d[interior_gt_bool], interior_pts)
 
 
-def point_inside_polygon_interior_sanity_check(n_vertices, poly_x_pts, poly_y_pts, test_x, test_y):
+def point_inside_polygon_interior_sanity_check(
+    n_vertices: int, poly_x_pts: np.ndarray, poly_y_pts: np.ndarray, test_x: float, test_y: float
+) -> bool:
     """
     We use this function to verify shapely.geometry's correctness. This fn only works correctly
     on the interior of an object (not on the boundary).
@@ -274,7 +276,7 @@ def point_inside_polygon_interior_sanity_check(n_vertices, poly_x_pts, poly_y_pt
     return (count % 2) == 1
 
 
-def test_point_in_polygon_shapely_vs_our_implementation():
+def test_point_in_polygon_shapely_vs_our_implementation() -> None:
     """
     Using 20 points originally sampled in unit square (uniform random),
     ensure that our implementation of point-in-polygon matches
@@ -346,7 +348,7 @@ def test_point_in_polygon_shapely_vs_our_implementation():
         assert inside == point_inside_polygon_interior_sanity_check(n_vertices, vert_x_pts, vert_y_pts, test_x, test_y)
 
 
-def test_point_in_polygon_square():
+def test_point_in_polygon_square() -> None:
     """
     Ensure point barely inside square boundary is "inside".
     """

From 2d4a490712ff950af0619a83cf8f5ee4c9d6a275 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 09:39:18 -0400
Subject: [PATCH 085/113] Typing fixes.

---
 sphinx/conf.py                 | 10 +++++-----
 tests/test_manhattan_search.py | 25 +++++++++++++++----------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/sphinx/conf.py b/sphinx/conf.py
index 4bd4cbce..9ffea2e4 100644
--- a/sphinx/conf.py
+++ b/sphinx/conf.py
@@ -41,13 +41,13 @@
     "sphinx_autodoc_typehints",
     "recommonmark",
 ]
-templates_path: List = []
-exclude_patterns: List = []
-source_suffix: Dict = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"}
+templates_path: List[str] = []
+exclude_patterns: List[str] = []
+source_suffix: Dict[str, str] = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"}
 
 # -- Options for HTML output -------------------------------------------------
 html_theme: str = "sphinx_rtd_theme"
-html_static_path: List = []
+html_static_path: List[str] = []
 
 
 # -- Extension configuration -------------------------------------------------
@@ -78,7 +78,7 @@
 # Extra configuration
 
 
-def setup(app):
+def setup(app: sphinx.application.Sphinx) -> None:
     app.add_config_value(
         "recommonmark_config",
         {
diff --git a/tests/test_manhattan_search.py b/tests/test_manhattan_search.py
index 5dc8b0c8..fef0e2e0 100644
--- a/tests/test_manhattan_search.py
+++ b/tests/test_manhattan_search.py
@@ -14,7 +14,7 @@
 )
 
 
-def assert_np_obj_arrs_eq(pruned_polygons: np.ndarray, gt_pruned_polygons: np.ndarray):
+def assert_np_obj_arrs_eq(pruned_polygons: np.ndarray, gt_pruned_polygons: np.ndarray) -> None:
     """Test for equivalency of two pology representations."""
     assert pruned_polygons.shape == gt_pruned_polygons.shape
     assert pruned_polygons.dtype == gt_pruned_polygons.dtype == object
@@ -31,13 +31,13 @@ def assert_np_obj_arrs_eq(pruned_polygons: np.ndarray, gt_pruned_polygons: np.nd
         (np.array([[-0.3, 0.5], [-0.3, 0.5], [-0.3, 0.5]]), np.array([-0.3, 0.5, -0.3, 0.5])),
         (np.array([[-0.3, 0.5, 50.1], [0.2, 0.1, -100.3], [-0.5, 1.9, -0.01]]), np.array([-0.5, 0.1, 0.2, 1.9])),
     ],
-)
-def test_compute_point_cloud_bbox_2d(point_cloud: np.ndarray, gt_bbox: np.ndarray):
+)  # type: ignore
+def test_compute_point_cloud_bbox_2d(point_cloud: np.ndarray, gt_bbox: np.ndarray) -> None:
     """Test for bounding box from pointcloud functionality."""
     assert np.allclose(compute_point_cloud_bbox(point_cloud), gt_bbox)
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def polygons_and_gt_bboxes() -> Tuple[List[np.ndarray], List[np.ndarray]]:
     """Return a list of polygons and the corresponding polygon bounding boxes."""
     poly_1 = np.array([[-1.5, -0.5], [0.5, -0.5], [-0.5, 1.5]])
@@ -57,7 +57,9 @@ def polygons_and_gt_bboxes() -> Tuple[List[np.ndarray], List[np.ndarray]]:
     return polygons, gt_poly_bboxes
 
 
-def test_find_all_polygon_bboxes_overlapping_query_bbox(polygons_and_gt_bboxes):
+def test_find_all_polygon_bboxes_overlapping_query_bbox(
+    polygons_and_gt_bboxes: Tuple[List[np.ndarray], List[np.ndarray]]
+) -> None:
     """Test for correctness of """
     poly_bboxes = np.array([compute_point_cloud_bbox(poly) for poly in polygons_and_gt_bboxes[0]])
 
@@ -68,7 +70,7 @@ def test_find_all_polygon_bboxes_overlapping_query_bbox(polygons_and_gt_bboxes):
     assert np.allclose(overlap_indxs, gt_overlap_indxs)
 
 
-def test_compute_polygon_bboxes(polygons_and_gt_bboxes):
+def test_compute_polygon_bboxes(polygons_and_gt_bboxes: Tuple[List[np.ndarray], List[np.ndarray]]) -> None:
     """Test for correctness of compute_polygon_bboxes."""
     polygon_bboxes = compute_polygon_bboxes(np.array(polygons_and_gt_bboxes[0], dtype=object))
     gt_polygon_bboxes = np.array(polygons_and_gt_bboxes[1])
@@ -78,10 +80,13 @@ def test_compute_polygon_bboxes(polygons_and_gt_bboxes):
 @pytest.mark.parametrize(
     "query_pt, query_search_range_manhattan, gt_indices",
     [(np.array([-0.5, 1.5]), 0.5, [0, 1, 3]), (np.array([-0.5, 1.5]), 0.499, [0, 3]), (np.array([0, 2]), 0.24, [])],
-)
+)  # type: ignore
 def test_prune_polygons_manhattan_dist_find_nearby(
-    query_pt: np.ndarray, query_search_range_manhattan: float, gt_indices: Sequence[int], polygons_and_gt_bboxes
-):
+    query_pt: np.ndarray,
+    query_search_range_manhattan: float,
+    gt_indices: Sequence[int],
+    polygons_and_gt_bboxes: Tuple[List[np.ndarray], List[np.ndarray]],
+) -> None:
     """Test for correctness of prune_polygons_manhattan_dist."""
     polygons = np.array(polygons_and_gt_bboxes[0], dtype=object)
     pruned_polygons = prune_polygons_manhattan_dist(query_pt, polygons.copy(), query_search_range_manhattan)
@@ -89,7 +94,7 @@ def test_prune_polygons_manhattan_dist_find_nearby(
     assert_np_obj_arrs_eq(gt_pruned_polygons, pruned_polygons)
 
 
-def test_find_local_polygons(polygons_and_gt_bboxes):
+def test_find_local_polygons(polygons_and_gt_bboxes: Tuple[List[np.ndarray], List[np.ndarray]]) -> None:
     """Test for correctness of find_local_polygons."""
     polygons = np.array(polygons_and_gt_bboxes[0], dtype=object)
     poly_bboxes = np.array(polygons_and_gt_bboxes[1])

From befc3c572edcd0bd119a3b67d4289161b83c2aaa Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 09:46:13 -0400
Subject: [PATCH 086/113] Typing fixes.

---
 argoverse/utils/subprocess_utils.py |  4 ++--
 tests/test_centerline_utils.py      | 10 +++++-----
 tests/test_ffmpeg_utils_unit.py     |  4 ++--
 tests/test_ply_loader.py            |  2 +-
 tests/test_subprocess_utils.py      |  4 ++--
 tests/test_vis_mask.py              |  6 +++---
 6 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/argoverse/utils/subprocess_utils.py b/argoverse/utils/subprocess_utils.py
index e9f99d83..22cab107 100644
--- a/argoverse/utils/subprocess_utils.py
+++ b/argoverse/utils/subprocess_utils.py
@@ -3,7 +3,7 @@
 from typing import Optional, Tuple
 
 
-def run_command(cmd: str, return_output: bool = False) -> Optional[Tuple[bytes, bytes]]:
+def run_command(cmd: str, return_output: bool = False) -> Tuple[Optional[bytes], Optional[bytes]]:
     """
     Block until system call completes
 
@@ -17,4 +17,4 @@ def run_command(cmd: str, return_output: bool = False) -> Optional[Tuple[bytes,
 
     if return_output:
         return stdout_data, stderr_data
-    return None
+    return None, None
diff --git a/tests/test_centerline_utils.py b/tests/test_centerline_utils.py
index f61f5546..da99bf1c 100644
--- a/tests/test_centerline_utils.py
+++ b/tests/test_centerline_utils.py
@@ -12,7 +12,7 @@
 )
 
 
-def temp_test_straight_centerline_to_polygon():
+def temp_test_straight_centerline_to_polygon() -> None:
     """
         Try converting a simple straight polyline into a polygon. Represents
         the conversion from a centerline to a lane segment polygon.
@@ -37,7 +37,7 @@ def temp_test_straight_centerline_to_polygon():
     assert np.array_equal(polygon, gt_polygon)
 
 
-def test_is_overlapping_lane_seq():
+def test_is_overlapping_lane_seq() -> None:
     """Test is_overlapping_lane_seq"""
 
     lane_seq1 = [1, 2, 3, 4]
@@ -56,7 +56,7 @@ def test_is_overlapping_lane_seq():
     assert not is_overlapping_lane_seq(lane_seq1, lane_seq2)
 
 
-def test_get_nt_distance_point():
+def test_get_nt_distance_point() -> None:
     """Compute distances in centerline frame for a point"""
     """Test Case
 
@@ -92,7 +92,7 @@ def test_get_nt_distance_point():
     assert_almost_equal(norm_dist, 1.000, 3)
 
 
-def test_get_nt_distance():
+def test_get_nt_distance() -> None:
     """Compute distances in centerline frame for a trajectory"""
     """Test Case
 
@@ -122,7 +122,7 @@ def test_get_nt_distance():
     np.array_equal(nt_dist, expected_nt_dist)
 
 
-def test_filter_candidate_centerlines():
+def test_filter_candidate_centerlines() -> None:
     """Test filter candidate centerlines"""
 
     # Test Case
diff --git a/tests/test_ffmpeg_utils_unit.py b/tests/test_ffmpeg_utils_unit.py
index 348d85fc..65edd2c1 100644
--- a/tests/test_ffmpeg_utils_unit.py
+++ b/tests/test_ffmpeg_utils_unit.py
@@ -4,7 +4,7 @@
 from argoverse.utils.ffmpeg_utils import write_nonsequential_idx_video, write_video
 
 
-def test_ffmpeg_seq_frame_vid_smokescreen():
+def test_ffmpeg_seq_frame_vid_smokescreen() -> None:
     """
         """
     image_prefix = "imgs_%d.jpg"
@@ -12,7 +12,7 @@ def test_ffmpeg_seq_frame_vid_smokescreen():
     write_video(image_prefix, output_prefix)
 
 
-def test_ffmpeg_nonseq_frame_vid_smokescreen():
+def test_ffmpeg_nonseq_frame_vid_smokescreen() -> None:
     """
         """
     img_wildcard = "imgs_%*.jpg"
diff --git a/tests/test_ply_loader.py b/tests/test_ply_loader.py
index c913406d..bbaa20db 100644
--- a/tests/test_ply_loader.py
+++ b/tests/test_ply_loader.py
@@ -9,7 +9,7 @@
 _TEST_DIR = pathlib.Path(__file__).parent
 
 
-def test_load_ply():
+def test_load_ply() -> None:
     ply_fpath = _TEST_DIR / "test_data/tracking/1/lidar/PC_0.ply"
     pc = load_ply(ply_fpath)
     pc_gt = np.array(
diff --git a/tests/test_subprocess_utils.py b/tests/test_subprocess_utils.py
index 3d72e91d..a2e2309d 100644
--- a/tests/test_subprocess_utils.py
+++ b/tests/test_subprocess_utils.py
@@ -5,7 +5,7 @@
 from argoverse.utils.subprocess_utils import run_command
 
 
-def test_run_command_smokescreen():
+def test_run_command_smokescreen() -> None:
     """
         Do not check output, just verify import works
         and does not crash.
@@ -14,7 +14,7 @@ def test_run_command_smokescreen():
     run_command(cmd)
 
 
-def test_run_command_cat():
+def test_run_command_cat() -> None:
     """
         Execute a command to dump a string to standard output. Returned
         output will be in byte format with a carriage return.
diff --git a/tests/test_vis_mask.py b/tests/test_vis_mask.py
index 61891d4c..5078653e 100644
--- a/tests/test_vis_mask.py
+++ b/tests/test_vis_mask.py
@@ -9,7 +9,7 @@
 from argoverse.visualization.vis_mask import decode_segment_to_mask, vis_mask, vis_one_image, vis_one_image_opencv
 
 
-def test_vis_mask():
+def test_vis_mask() -> None:
     # Ordered Z first for easy reading
     img = np.array(
         [
@@ -39,7 +39,7 @@ def test_vis_mask():
     assert (expected_img == masked_image).all()
 
 
-def test_decode_segment_to_mask():
+def test_decode_segment_to_mask() -> None:
     mask = decode_segment_to_mask((2, 1, 3, 3), np.zeros((3, 3, 3)))
 
     expected_mask = np.array([[0, 0, 0], [0, 0, 1], [0, 0, 1]])
@@ -47,7 +47,7 @@ def test_decode_segment_to_mask():
     assert (mask == expected_mask).all()
 
 
-def mask_vis_unit_test():
+def mask_vis_unit_test() -> None:
     unit_test_dir = "test_data/vis_mask/0d2ee2db-4061-36b2-a330-8301bdce3fe8/00035"
     img_fpath = f"{unit_test_dir}/image_raw_ring_side_left_000000035.jpg"
     img = cv2.imread(img_fpath)

From 538d2580d95a62f344b70c8d5f7450d81bc997e3 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 10:29:45 -0400
Subject: [PATCH 087/113] More typing fixes.

---
 argoverse/map_representation/lane_segment.py |   6 +-
 argoverse/map_representation/map_api.py      |  18 +--
 argoverse/utils/centerline_utils.py          |   4 +-
 integration_tests/test_map_api.py            | 135 +++++++++++--------
 integration_tests/test_mayavi_utils.py       |  33 ++---
 tests/test_cv2_plotting_utils.py             |   6 +-
 tests/test_eval_tracking.py                  |  18 +--
 tests/test_mpl_point_cloud_vis.py            |   2 +-
 tests/test_polyline_density.py               |   2 +-
 9 files changed, 124 insertions(+), 100 deletions(-)

diff --git a/argoverse/map_representation/lane_segment.py b/argoverse/map_representation/lane_segment.py
index dfd570ff..07616f99 100644
--- a/argoverse/map_representation/lane_segment.py
+++ b/argoverse/map_representation/lane_segment.py
@@ -1,5 +1,5 @@
 # 
-from typing import Optional, Sequence
+from typing import List, Optional, Sequence
 
 import numpy as np
 
@@ -13,8 +13,8 @@ def __init__(
         is_intersection: bool,
         l_neighbor_id: Optional[int],
         r_neighbor_id: Optional[int],
-        predecessors: Sequence[int],
-        successors: Optional[Sequence[int]],
+        predecessors: List[int],
+        successors: Optional[List[int]],
         centerline: np.ndarray,
     ) -> None:
         """Initialize the lane segment.
diff --git a/argoverse/map_representation/map_api.py b/argoverse/map_representation/map_api.py
index 38e8e391..2a28a8d2 100644
--- a/argoverse/map_representation/map_api.py
+++ b/argoverse/map_representation/map_api.py
@@ -2,7 +2,7 @@
 
 import copy
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -620,7 +620,7 @@ def get_lane_ids_in_xy_bbox(
 
         return neighborhood_lane_ids
 
-    def get_lane_segment_predecessor_ids(self, lane_segment_id: int, city_name: str) -> Sequence[int]:
+    def get_lane_segment_predecessor_ids(self, lane_segment_id: int, city_name: str) -> List[int]:
         """
         Get land id for the lane predecessor of the specified lane_segment_id
 
@@ -634,7 +634,7 @@ def get_lane_segment_predecessor_ids(self, lane_segment_id: int, city_name: str)
         predecessor_ids = self.city_lane_centerlines_dict[city_name][lane_segment_id].predecessors
         return predecessor_ids
 
-    def get_lane_segment_successor_ids(self, lane_segment_id: int, city_name: str) -> Optional[Sequence[int]]:
+    def get_lane_segment_successor_ids(self, lane_segment_id: int, city_name: str) -> Optional[List[int]]:
         """
         Get land id for the lane sucessor of the specified lane_segment_id
 
@@ -741,20 +741,20 @@ def lane_has_traffic_control_measure(self, lane_segment_id: int, city_name: str)
         return self.city_lane_centerlines_dict[city_name][lane_segment_id].has_traffic_control
 
     def remove_extended_predecessors(
-        self, lane_seqs: List[Sequence[int]], xy: np.ndarray, city_name: str
-    ) -> List[Sequence[int]]:
+        self, lane_seqs: List[List[int]], xy: np.ndarray, city_name: str
+    ) -> List[List[int]]:
         """
         Remove lane_ids which are obtained by finding way too many predecessors from lane sequences.
         If any lane id is an occupied lane id for the first coordinate of the trajectory, ignore all the
         lane ids that occured before that
 
         Args:
-            lane_seqs: List of sequence of lane ids (Eg. [[12345, 12346, 12347], [12345, 12348]])
+            lane_seqs: List of list of lane ids (Eg. [[12345, 12346, 12347], [12345, 12348]])
             xy: trajectory coordinates
             city_name: either 'MIA' for Miami or 'PIT' for Pittsburgh
 
         Returns:
-            filtered_lane_seq (list of list of integers): List of sequence of lane ids obtained after filtering
+            filtered_lane_seq (list of list of integers): List of list of lane ids obtained after filtering
         """
         filtered_lane_seq = []
         occupied_lane_ids = self.get_lane_segments_containing_xy(xy[0, 0], xy[0, 1], city_name)
@@ -767,7 +767,7 @@ def remove_extended_predecessors(
             filtered_lane_seq.append(new_lane_seq)
         return filtered_lane_seq
 
-    def get_cl_from_lane_seq(self, lane_seqs: Iterable[Sequence[int]], city_name: str) -> List[np.ndarray]:
+    def get_cl_from_lane_seq(self, lane_seqs: Iterable[List[int]], city_name: str) -> List[np.ndarray]:
         """Get centerlines corresponding to each lane sequence in lane_sequences
 
         Args:
@@ -822,7 +822,7 @@ def get_candidate_centerlines_for_traj(
         dfs_threshold = displacement * 2.0
 
         # DFS to get all successor and predecessor candidates
-        obs_pred_lanes: List[Sequence[int]] = []
+        obs_pred_lanes: List[List[int]] = []
         for lane in curr_lane_candidates:
             candidates_future = self.dfs(lane, city_name, 0, dfs_threshold)
             candidates_past = self.dfs(lane, city_name, 0, dfs_threshold, True)
diff --git a/argoverse/utils/centerline_utils.py b/argoverse/utils/centerline_utils.py
index d5eafe5c..1c6fd770 100644
--- a/argoverse/utils/centerline_utils.py
+++ b/argoverse/utils/centerline_utils.py
@@ -350,12 +350,12 @@ def get_centerlines_most_aligned_with_trajectory(xy: np.ndarray, candidate_cl: L
     return candidate_centerlines
 
 
-def remove_overlapping_lane_seq(lane_seqs: Sequence[Sequence[int]]) -> List[Sequence[int]]:
+def remove_overlapping_lane_seq(lane_seqs: List[List[int]]) -> List[List[int]]:
     """
     Remove lane sequences which are overlapping to some extent
 
     Args:
-        lane_seqs (list of list of integers): List of sequence of lane ids (Eg. [[12345, 12346, 12347], [12345, 12348]])
+        lane_seqs (list of list of integers): List of list of lane ids (Eg. [[12345, 12346, 12347], [12345, 12348]])
 
     Returns:
         List of sequence of lane ids (e.g. ``[[12345, 12346, 12347], [12345, 12348]]``)
diff --git a/integration_tests/test_map_api.py b/integration_tests/test_map_api.py
index 95a136d3..5e8ab2ac 100644
--- a/integration_tests/test_map_api.py
+++ b/integration_tests/test_map_api.py
@@ -2,7 +2,9 @@
 """Map API unit tests"""
 
 import glob
+from typing import Set, Tuple
 
+import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
 
@@ -13,13 +15,24 @@
 from argoverse.utils.mpl_plotting_utils import plot_lane_segment_patch
 
 
-def add_lane_segment_to_ax(ax, lane_centerline, lane_polygon, patch_color, xmin, xmax, ymin, ymax):
+def add_lane_segment_to_ax(
+    ax: plt.axes.Axis,
+    lane_centerline: np.ndarray,
+    lane_polygon: np.ndarray,
+    patch_color: str,
+    xmin: float,
+    xmax: float,
+    ymin: float,
+    ymax: float,
+) -> None:
     """
         """
     plot_lane_segment_patch(lane_polygon, ax, color=patch_color, alpha=0.3)
 
 
-def find_lane_segment_bounds_in_table(adm, city_name, lane_segment_id):
+def find_lane_segment_bounds_in_table(
+    avm: ArgoverseMap, city_name: str, lane_segment_id: int
+) -> Tuple[float, float, float, float]:
     """
         """
     match_found = False
@@ -35,7 +48,7 @@ def find_lane_segment_bounds_in_table(adm, city_name, lane_segment_id):
     return xmin, ymin, xmax, ymax
 
 
-def verify_halluc_lane_extent_index(enable_lane_boundaries=False):
+def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> None:
     """
 
         """
@@ -47,7 +60,7 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries=False):
         # get all lane segment IDs inside of this city
         lane_segment_ids = list(avm.city_lane_centerlines_dict[city_name].keys())
         for lane_segment_id in lane_segment_ids:
-            xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, lane_segment_id)
+            xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, lane_segment_id)
 
             predecessor_ids = avm.get_lane_segment_predecessor_ids(lane_segment_id, city_name)
             successor_ids = avm.get_lane_segment_successor_ids(lane_segment_id, city_name)
@@ -66,7 +79,7 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries=False):
                 for predecessor_id in predecessor_ids:
                     lane_centerline = avm.get_lane_segment_centerline(predecessor_id, city_name)
                     halluc_lane_polygon = avm.get_lane_segment_polygon(predecessor_id, city_name)
-                    xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, predecessor_id)
+                    xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, predecessor_id)
                     add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "r", xmin, xmax, ymin, ymax)
 
             if successor_ids is not None:
@@ -74,21 +87,21 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries=False):
                 for successor_id in successor_ids:
                     lane_centerline = avm.get_lane_segment_centerline(successor_id, city_name)
                     halluc_lane_polygon = avm.get_lane_segment_polygon(successor_id, city_name)
-                    xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, successor_id)
+                    xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, successor_id)
                     add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "b", xmin, xmax, ymin, ymax)
 
             # add left neighbor
             if l_neighbor_id is not None:
                 lane_centerline = avm.get_lane_segment_centerline(l_neighbor_id, city_name)
                 halluc_lane_polygon = avm.get_lane_segment_polygon(l_neighbor_id, city_name)
-                xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, l_neighbor_id)
+                xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, l_neighbor_id)
                 add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "g", xmin, xmax, ymin, ymax)
 
             # add right neighbor
             if r_neighbor_id is not None:
                 lane_centerline = avm.get_lane_segment_centerline(r_neighbor_id, city_name)
                 halluc_lane_polygon = avm.get_lane_segment_polygon(r_neighbor_id, city_name)
-                xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, r_neighbor_id)
+                xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, r_neighbor_id)
                 add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "m", xmin, xmax, ymin, ymax)
 
             if enable_lane_boundaries:
@@ -106,58 +119,58 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries=False):
             plt.close("all")
 
 
-def verify_manhattan_search_functionality():
-    """
-        Minimal example where we
-        """
-    adm = ArgoverseMap()
-    # query_x = 254.
-    # query_y = 1778.
+# def verify_manhattan_search_functionality() -> None:
+#     """
+#         Minimal example where we
+#         """
+#     avm = ArgoverseMap()
+#     # query_x = 254.
+#     # query_y = 1778.
 
-    ref_query_x = 422.0
-    ref_query_y = 1005.0
+#     ref_query_x = 422.0
+#     ref_query_y = 1005.0
 
-    city_name = "PIT"  # 'MIA'
-    for trial_idx in range(10):
-        query_x = ref_query_x + (np.random.rand() - 0.5) * 10
-        query_y = ref_query_y + (np.random.rand() - 0.5) * 10
+#     city_name = "PIT"  # 'MIA'
+#     for trial_idx in range(10):
+#         query_x = ref_query_x + (np.random.rand() - 0.5) * 10
+#         query_y = ref_query_y + (np.random.rand() - 0.5) * 10
 
-        # query_x,query_y = (3092.49845414,1798.55426805)
-        query_x, query_y = (3112.80160113, 1817.07585338)
+#         # query_x,query_y = (3092.49845414,1798.55426805)
+#         query_x, query_y = (3112.80160113, 1817.07585338)
 
-        lane_segment_ids = avm.get_lane_ids_in_xy_bbox(query_x, query_y, city_name, 5000)
+#         lane_segment_ids = avm.get_lane_ids_in_xy_bbox(query_x, query_y, city_name, 5000)
 
-        fig = plt.figure(figsize=(22.5, 8))
-        ax = fig.add_subplot(111)
-        # ax.scatter([query_x], [query_y], 500, color='k', marker='.')
+#         fig = plt.figure(figsize=(22.5, 8))
+#         ax = fig.add_subplot(111)
+#         # ax.scatter([query_x], [query_y], 500, color='k', marker='.')
 
-        plot_lane_segment_patch(pittsburgh_bounds, ax, color="m", alpha=0.1)
+#         plot_lane_segment_patch(pittsburgh_bounds, ax, color="m", alpha=0.1)
 
-        if len(lane_segment_ids) > 0:
-            for i, lane_segment_id in enumerate(lane_segment_ids):
-                patch_color = "y"  # patch_colors[i % 4]
-                lane_centerline = avm.get_lane_segment_centerline(lane_segment_id, city_name)
+#         if len(lane_segment_ids) > 0:
+#             for i, lane_segment_id in enumerate(lane_segment_ids):
+#                 patch_color = "y"  # patch_colors[i % 4]
+#                 lane_centerline = avm.get_lane_segment_centerline(lane_segment_id, city_name)
 
-                test_x, test_y = lane_centerline.mean(axis=0)
-                inside = point_inside_polygon(
-                    n_poly_vertices, pittsburgh_bounds[:, 0], pittsburgh_bounds[:, 1], test_x, test_y
-                )
+#                 test_x, test_y = lane_centerline.mean(axis=0)
+#                 inside = point_inside_polygon(
+#                     n_poly_vertices, pittsburgh_bounds[:, 0], pittsburgh_bounds[:, 1], test_x, test_y
+#                 )
 
-                if inside:
-                    halluc_lane_polygon = avm.get_lane_segment_polygon(lane_segment_id, city_name)
-                    xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(adm, city_name, lane_segment_id)
-                    add_lane_segment_to_ax(
-                        ax, lane_centerline, halluc_lane_polygon, patch_color, xmin, xmax, ymin, ymax
-                    )
+#                 if inside:
+#                     halluc_lane_polygon = avm.get_lane_segment_polygon(lane_segment_id, city_name)
+#                     xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, lane_segment_id)
+#                     add_lane_segment_to_ax(
+#                         ax, lane_centerline, halluc_lane_polygon, patch_color, xmin, xmax, ymin, ymax
+#                     )
 
-        ax.axis("equal")
-        plt.show()
-        datetime_str = generate_datetime_string()
-        plt.savefig(f"{trial_idx}_{datetime_str}.jpg")
-        plt.close("all")
+#         ax.axis("equal")
+#         plt.show()
+#         datetime_str = generate_datetime_string()
+#         plt.savefig(f"{trial_idx}_{datetime_str}.jpg")
+#         plt.close("all")
 
 
-def verify_point_in_polygon_for_lanes():
+def verify_point_in_polygon_for_lanes() -> None:
     """
         """
     avm = ArgoverseMap()
@@ -183,8 +196,8 @@ def verify_point_in_polygon_for_lanes():
             plot_lane_segment_patch(halluc_lane_polygon, ax, color="y", alpha=0.3)
 
         nearby_lane_ids = avm.get_lane_ids_in_xy_bbox(query_x, query_y, city_name)
-        nearby_lane_ids = set(nearby_lane_ids) - set(occupied_lane_ids)
-        for nearby_lane_id in nearby_lane_ids:
+        nearby_unoccupied_lane_ids: Set[int] = set(nearby_lane_ids) - set(occupied_lane_ids)
+        for nearby_lane_id in nearby_unoccupied_lane_ids:
             halluc_lane_polygon = avm.get_lane_segment_polygon(nearby_lane_id, city_name)
             plot_lane_segment_patch(halluc_lane_polygon, ax, color="r", alpha=0.3)
 
@@ -193,7 +206,15 @@ def verify_point_in_polygon_for_lanes():
         plt.close("all")
 
 
-def plot_nearby_halluc_lanes(ax, city_name, adm, query_x, query_y, patch_color="r", radius=20):
+def plot_nearby_halluc_lanes(
+    ax: plt.axes.Axis,
+    city_name: str,
+    avm: ArgoverseMap,
+    query_x: float,
+    query_y: float,
+    patch_color: str = "r",
+    radius: float = 20.0,
+) -> None:
     """
         """
     nearby_lane_ids = avm.get_lane_ids_in_xy_bbox(query_x, query_y, city_name, radius)
@@ -203,7 +224,7 @@ def plot_nearby_halluc_lanes(ax, city_name, adm, query_x, query_y, patch_color="
         plt.text(halluc_lane_polygon[:, 0].mean(), halluc_lane_polygon[:, 1].mean(), str(nearby_lane_id))
 
 
-def verify_lane_tangent_vector():
+def verify_lane_tangent_vector() -> None:
     """
         debug low confidence lane tangent predictions
 
@@ -215,7 +236,7 @@ def verify_lane_tangent_vector():
     # both of these are Pittsburgh logs
     log_ids = ["033669d3-3d6b-3d3d-bd93-7985d86653ea", "028d5cb1-f74d-366c-85ad-84fde69b0fd3"]
 
-    adm = ArgoverseMap()
+    avm = ArgoverseMap()
     city_name = "PIT"
     for log_id in log_ids:
         print(f"On {log_id}")
@@ -258,14 +279,14 @@ def verify_lane_tangent_vector():
                 ax.scatter([query_x], [query_y], 100, color="k", marker=".")
                 # make another plot now!
 
-                plot_nearby_halluc_lanes(ax, city_name, adm, query_x, query_y)
+                plot_nearby_halluc_lanes(ax, city_name, avm, query_x, query_y)
 
                 ax.axis("equal")
                 plt.show()
                 plt.close("all")
 
 
-def test_remove_extended_predecessors():
+def test_remove_extended_predecessors() -> None:
     """Test remove_extended_predecessors() for map_api"""
 
     lane_seqs = [[9621385, 9619110, 9619209, 9631133], [9621385, 9619110, 9619209], [9619209, 9631133]]
@@ -280,7 +301,7 @@ def test_remove_extended_predecessors():
     ), "remove_extended_predecessors() failed!"
 
 
-def test_get_candidate_centerlines_for_traj():
+def test_get_candidate_centerlines_for_traj() -> None:
     """Test get_candidate_centerlines_for_traj()
 
     -180        .  .  .  .  .                   -100
@@ -426,7 +447,7 @@ def test_get_candidate_centerlines_for_traj():
         assert np.allclose(expected_centerlines[i], candidate_centerlines[i]), "Centerline coordinates wrong!"
 
 
-def test_dfs():
+def test_dfs() -> None:
     """Test dfs for lane graph
 
     Lane Graph:
diff --git a/integration_tests/test_mayavi_utils.py b/integration_tests/test_mayavi_utils.py
index b637f675..33a6adc7 100644
--- a/integration_tests/test_mayavi_utils.py
+++ b/integration_tests/test_mayavi_utils.py
@@ -16,17 +16,18 @@
     MISSING_MAYAVI = True
 
 import pathlib
+from pathlib import Path
 
 import numpy as np
 import pytest
 
-_TEST_DIR = pathlib.Path(__file__).parent.parent / "tests"
+_TEST_DIR: Path = pathlib.Path(__file__).parent.parent / "tests"
 
-skip_if_not_mayavi = pytest.mark.skipif(MISSING_MAYAVI, reason="mayavi not installed")
+skip_if_not_mayavi: pytest.mark.MarkDecorator = pytest.mark.skipif(MISSING_MAYAVI, reason="mayavi not installed")
 
 
-@skip_if_not_mayavi
-def test_mayavi_import_basic():
+@skip_if_not_mayavi  # type: ignore
+def test_mayavi_import_basic() -> None:
     """
         To test if Mayavi is installed correctly, generate lines around
         surface of a torus and render them.
@@ -43,8 +44,8 @@ def test_mayavi_import_basic():
     l = mayavi.mlab.plot3d(x, y, z, np.sin(mu), tube_radius=0.025, colormap="Spectral")
 
 
-@skip_if_not_mayavi
-def test_plot_bbox_3d_mayavi_no_text():
+@skip_if_not_mayavi  # type: ignore
+def test_plot_bbox_3d_mayavi_no_text() -> None:
     """
     To visualize the result, simply add the following line to this function:
         mayavi.mlab.show()
@@ -96,8 +97,8 @@ def test_plot_bbox_3d_mayavi_no_text():
     mayavi.mlab.close(fig)
 
 
-@skip_if_not_mayavi
-def test_plot_bbox_3d_mayavi_drawtext():
+@skip_if_not_mayavi  # type: ignore
+def test_plot_bbox_3d_mayavi_drawtext() -> None:
     """
     To visualize the result, simply add the following line to this function:
     .. code-block:: python
@@ -153,8 +154,8 @@ def test_plot_bbox_3d_mayavi_drawtext():
     mayavi.mlab.close(fig)
 
 
-@skip_if_not_mayavi
-def test_plot_points_3D_mayavi():
+@skip_if_not_mayavi  # type: ignore
+def test_plot_points_3D_mayavi() -> None:
     """Visualize 3D point cloud with Mayavi.
 
     Note
@@ -174,8 +175,8 @@ def test_plot_points_3D_mayavi():
     mayavi.mlab.close(fig)
 
 
-@skip_if_not_mayavi
-def test_plot_points_3d_argoverse():
+@skip_if_not_mayavi  # type: ignore
+def test_plot_points_3d_argoverse() -> None:
     """Render a LiDAR sweep from Argoverse, loaded from a .txt file."""
     fig = mayavi.mlab.figure(bgcolor=(1, 1, 1), size=(2000, 1000))
     point_arr = np.loadtxt(_TEST_DIR / "test_data/sample_argoverse_sweep.txt")
@@ -183,16 +184,16 @@ def test_plot_points_3d_argoverse():
     mayavi.mlab.close(fig)
 
 
-@skip_if_not_mayavi
-def test_draw_lidar_argoverse():
+@skip_if_not_mayavi  # type: ignore
+def test_draw_lidar_argoverse() -> None:
     """Test :ref:`draw_lidar_simple`."""
     pc = np.loadtxt(_TEST_DIR / "test_data/sample_argoverse_sweep.txt")
     fig = draw_lidar(pc)
     mayavi.mlab.close(fig)
 
 
-@skip_if_not_mayavi
-def test_draw_coordinate_frame_at_origin():
+@skip_if_not_mayavi  # type: ignore
+def test_draw_coordinate_frame_at_origin() -> None:
     """Test :ref:`draw_coordinate_frame_at_origin`.
     """
     fig = mayavi.mlab.figure(bgcolor=(1, 1, 1), size=(2000, 1000))
diff --git a/tests/test_cv2_plotting_utils.py b/tests/test_cv2_plotting_utils.py
index ff219891..b02f2b10 100644
--- a/tests/test_cv2_plotting_utils.py
+++ b/tests/test_cv2_plotting_utils.py
@@ -10,7 +10,7 @@
 )
 
 
-def test_draw_point_cloud_in_img_cv2_smokescreen():
+def test_draw_point_cloud_in_img_cv2_smokescreen() -> None:
     """
         We place 4 red circles in an image (with channel order BGR,
         per the OpenCV convention). We verify that pixel values change accordingly.
@@ -34,7 +34,7 @@ def test_draw_point_cloud_in_img_cv2_smokescreen():
             assert np.allclose(img_w_circ[y, x, :], color)
 
 
-def test_draw_polygon_cv2_smokescreen():
+def test_draw_polygon_cv2_smokescreen() -> None:
     """
         Test ability to fill a nonconvex polygon.
 
@@ -64,7 +64,7 @@ def test_draw_polygon_cv2_smokescreen():
         assert img_w_polygon.dtype == dtype
 
 
-def test_plot_bbox_polygon_cv2_smokescreen():
+def test_plot_bbox_polygon_cv2_smokescreen() -> None:
     """
         Test drawing a green bounding box, with a thin red border, and
         test plotted inside, representing a track ID.
diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py
index a7ac86ae..c860ba94 100644
--- a/tests/test_eval_tracking.py
+++ b/tests/test_eval_tracking.py
@@ -73,7 +73,7 @@ class TrackedObjRec(NamedTuple):
 class TrackedObjects:
     def __init__(self, log_id: str, is_gt: bool) -> None:
         """ """
-        self.ts_to_trackedlabels_dict: DefaultDict[int, List[Dict]] = defaultdict(list)
+        self.ts_to_trackedlabels_dict: DefaultDict[int, List[Dict[str, Any]]] = defaultdict(list)
         self.log_id = log_id
 
         tracks_type = "gt" if is_gt else "pred"
@@ -110,7 +110,9 @@ def save_to_disk(self) -> None:
             save_json_dict(json_fpath, ts_trackedlabels)
 
 
-def dump_1obj_scenario_json(centers, yaw_angles, log_id: str, is_gt: bool) -> None:
+def dump_1obj_scenario_json(
+    centers: List[Tuple[int, int, int]], yaw_angles: List[float], log_id: str, is_gt: bool
+) -> None:
     """
 	Egovehicle stationary (represented by `o`).
 	Sequence of 4-nanosecond timestamps.
@@ -178,7 +180,7 @@ def run_eval(exp_name: str) -> Mapping[str, Any]:
     return result_dict
 
 
-def get_1obj_gt_scenario():
+def get_1obj_gt_scenario() -> Tuple[List[Tuple[int, int, int]], List[float]]:
     """
 	Egovehicle stationary (represented by `o`).
 	Seqeuence of 4-nanosecond timestamps.
@@ -224,7 +226,7 @@ def get_1obj_gt_scenario():
     cz = 0
     centers += [(cx, cy, cz)]
 
-    yaw_angles = [0, 0, 0, 0]
+    yaw_angles = [0.0, 0.0, 0.0, 0.0]
     return centers, yaw_angles
 
 
@@ -285,7 +287,7 @@ def test_1obj_offset_translation() -> None:
     cz = 0
     centers += [(cx, cy, cz)]
 
-    yaw_angles = [0, 0, 0, 0]
+    yaw_angles = [0.0, 0.0, 0.0, 0.0]
 
     # dump the ground truth first
     gt_centers, gt_yaw_angles = get_1obj_gt_scenario()
@@ -351,7 +353,7 @@ def test_1obj_poor_translation() -> None:
     cz = 0
     centers += [(cx, cy, cz)]
 
-    yaw_angles = [0, 0, 0, 0]
+    yaw_angles = [0.0, 0.0, 0.0, 0.0]
 
     # dump the ground truth first
     gt_centers, gt_yaw_angles = get_1obj_gt_scenario()
@@ -508,7 +510,7 @@ def test_orientation_error8() -> None:
     assert np.allclose(error_deg, 2.0, atol=1e-2)
 
 
-def get_mot16_scenario_a():
+def get_mot16_scenario_a() -> Tuple[List[Tuple[int, int, int]], List[float]]:
     """
 	https://arxiv.org/pdf/1603.00831.pdf
 	"""
@@ -549,7 +551,7 @@ def get_mot16_scenario_a():
     cz = 0
     centers += [(cx, cy, cz)]
 
-    yaw_angles = [0, 0, 0, 0, 0, 0]
+    yaw_angles = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
     return centers, yaw_angles
 
 
diff --git a/tests/test_mpl_point_cloud_vis.py b/tests/test_mpl_point_cloud_vis.py
index 2647b773..c913dcfa 100644
--- a/tests/test_mpl_point_cloud_vis.py
+++ b/tests/test_mpl_point_cloud_vis.py
@@ -11,7 +11,7 @@
 _TEST_DIR = pathlib.Path(__file__).parent
 
 
-def test_draw_point_cloud_bev_smokescreen():
+def test_draw_point_cloud_bev_smokescreen() -> None:
     """Test :ref:`draw_point_cloud_bev`"""
     fig3d = plt.figure(figsize=(15, 8))
     ax_bev = fig3d.add_subplot(111)
diff --git a/tests/test_polyline_density.py b/tests/test_polyline_density.py
index ff3e4019..4deb1cdf 100644
--- a/tests/test_polyline_density.py
+++ b/tests/test_polyline_density.py
@@ -5,7 +5,7 @@
 from argoverse.utils import polyline_density
 
 
-def test_polyline_length():
+def test_polyline_length() -> None:
     line = np.array([[0, 0], [0, 1], [1, 1], [1, 0], [2, 0], [2, 1]])
 
     length = polyline_density.get_polyline_length(line)

From 7c7fff0c4e6b4c51755c1502a4f057fde05a1a0b Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 11:19:30 -0400
Subject: [PATCH 088/113] More type fixes.

---
 .../argoverse_forecasting_loader.py           |  6 +-
 argoverse/utils/plane_visualization_utils.py  |  6 +-
 integration_tests/conftest.py                 |  6 +-
 integration_tests/test_eval_forecasting.py    |  2 +-
 integration_tests/test_mayavi_utils.py        |  2 +-
 .../test_plane_visualization_utils.py         | 16 ++--
 integration_tests/test_tracker_eval.py        | 76 +++++++++----------
 tests/conftest.py                             |  5 +-
 tests/test_argoverse_forecasting_loader.py    | 20 +++--
 tests/test_argoverse_tracking_loader.py       | 16 ++--
 tests/test_eval_detection.py                  | 23 ++++--
 tests/test_frame_label_accumulator.py         |  5 +-
 tests/test_mayavi_utils_when_missing.py       |  2 +-
 tests/test_simple_track_dataloader.py         | 10 ++-
 tests/test_visualization_utils.py             | 12 +--
 15 files changed, 117 insertions(+), 90 deletions(-)

diff --git a/argoverse/data_loading/argoverse_forecasting_loader.py b/argoverse/data_loading/argoverse_forecasting_loader.py
index 535267ed..fb8da5dc 100644
--- a/argoverse/data_loading/argoverse_forecasting_loader.py
+++ b/argoverse/data_loading/argoverse_forecasting_loader.py
@@ -2,7 +2,7 @@
 import os
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Mapping, Optional, Sequence, Union
+from typing import Any, List, Sequence, Union
 
 import numpy as np
 import pandas as pd
@@ -39,13 +39,13 @@ def __init__(self, root_dir: Union[str, Path]):
         self.current_seq: Path = self.seq_list[self.counter]
 
     @property
-    def track_id_list(self) -> Sequence[int]:
+    def track_id_list(self) -> List[int]:
         """Get the track ids in the current sequence.
 
         Returns:
             list of track ids in the current sequence
         """
-        _track_id_list: Sequence[int] = np.unique(self.seq_df["TRACK_ID"].values).tolist()
+        _track_id_list: List[int] = np.unique(self.seq_df["TRACK_ID"].values).tolist()
         return _track_id_list
 
     @property
diff --git a/argoverse/utils/plane_visualization_utils.py b/argoverse/utils/plane_visualization_utils.py
index 36d9fbf8..978e8dcf 100644
--- a/argoverse/utils/plane_visualization_utils.py
+++ b/argoverse/utils/plane_visualization_utils.py
@@ -1,7 +1,7 @@
 # 
 
 import sys
-from typing import List, Optional, Sequence
+from typing import List, Optional
 
 import numpy as np
 
@@ -16,7 +16,7 @@
 )
 
 
-def populate_frustum_voxels(planes: Sequence[np.ndarray], fig: Figure, axis_pair: str) -> Figure:
+def populate_frustum_voxels(planes: List[np.ndarray], fig: Figure, axis_pair: str) -> Figure:
     """
     Generate grid in xy plane, and then treat it as grid in xz (ground) plane
     in camera coordinate system.
@@ -52,7 +52,7 @@ def populate_frustum_voxels(planes: Sequence[np.ndarray], fig: Figure, axis_pair
 
 
 def plot_frustum_planes_and_normals(
-    planes: Sequence[np.ndarray], cuboid_verts: Optional[np.ndarray] = None, near_clip_dist: float = 0.5
+    planes: List[np.ndarray], cuboid_verts: Optional[np.ndarray] = None, near_clip_dist: float = 0.5
 ) -> None:
     """
     Args:
diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
index 97f62b41..878920e4 100644
--- a/integration_tests/conftest.py
+++ b/integration_tests/conftest.py
@@ -7,11 +7,11 @@
 import pytest
 
 
-@pytest.fixture(autouse=True)
-def set_log_level(caplog):
+@pytest.fixture(autouse=True)  # type: ignore
+def set_log_level(caplog: logging.Logger) -> None:
     """Set the log level.
 
     Set the log level to DEBUG for our testing to make sure that any bad log
     statements throw errors.
     """
-    caplog.set_level(logging.DEBUG)
+    caplog.setLevel(logging.DEBUG)
diff --git a/integration_tests/test_eval_forecasting.py b/integration_tests/test_eval_forecasting.py
index b4ba22d4..5131e66f 100644
--- a/integration_tests/test_eval_forecasting.py
+++ b/integration_tests/test_eval_forecasting.py
@@ -8,7 +8,7 @@
 from argoverse.evaluation.eval_forecasting import compute_forecasting_metrics
 
 
-def test_compute_forecasting_metric():
+def test_compute_forecasting_metric() -> None:
     """Test computation of motion forecasting metrics."""
     # Test Case:
 
diff --git a/integration_tests/test_mayavi_utils.py b/integration_tests/test_mayavi_utils.py
index 33a6adc7..ae19f61d 100644
--- a/integration_tests/test_mayavi_utils.py
+++ b/integration_tests/test_mayavi_utils.py
@@ -23,7 +23,7 @@
 
 _TEST_DIR: Path = pathlib.Path(__file__).parent.parent / "tests"
 
-skip_if_not_mayavi: pytest.mark.MarkDecorator = pytest.mark.skipif(MISSING_MAYAVI, reason="mayavi not installed")
+skip_if_not_mayavi = pytest.mark.skipif(MISSING_MAYAVI, reason="mayavi not installed")
 
 
 @skip_if_not_mayavi  # type: ignore
diff --git a/integration_tests/test_plane_visualization_utils.py b/integration_tests/test_plane_visualization_utils.py
index 4be8292b..76f4c11c 100644
--- a/integration_tests/test_plane_visualization_utils.py
+++ b/integration_tests/test_plane_visualization_utils.py
@@ -28,6 +28,7 @@
 if not MAYAVI_MISSING:
     calib = load_calib(os.fspath(_TEST_DIR / "test_data/tracking/1/vehicle_calibration_info.json"))["ring_front_center"]
     planes = generate_frustum_planes(calib.K, calib.camera)
+    assert planes is not None
 
 
 skip_if_mayavi_missing = pytest.mark.skipif(
@@ -35,24 +36,27 @@
 )
 
 
-@skip_if_mayavi_missing
-def test_plot_frustum_planes_and_normals():
+@skip_if_mayavi_missing  # type: ignore
+def test_plot_frustum_planes_and_normals() -> None:
     """
     """
 
+    assert planes is not None
     plot_frustum_planes_and_normals(planes, cuboid_verts=None, near_clip_dist=0.5)
 
 
-@skip_if_mayavi_missing
-def test_populate_frustum_voxels():
+@skip_if_mayavi_missing  # type: ignore
+def test_populate_frustum_voxels() -> None:
     """
     """
     fig, axis_pair = plt.subplots(1, 1, figsize=(20, 15))
+
+    assert planes is not None
     populate_frustum_voxels(planes, fig, axis_pair)
 
 
-@skip_if_mayavi_missing
-def test_get_perpendicular():
+@skip_if_mayavi_missing  # type: ignore
+def test_get_perpendicular() -> None:
     """
     """
     n = [0, 1, 1]
diff --git a/integration_tests/test_tracker_eval.py b/integration_tests/test_tracker_eval.py
index fded64c6..5bb17953 100644
--- a/integration_tests/test_tracker_eval.py
+++ b/integration_tests/test_tracker_eval.py
@@ -16,7 +16,7 @@
 D_MAX = 100
 
 
-def test_in_distance_range():
+def test_in_distance_range() -> None:
     """
         test for in_distance_range_pose()
         """
@@ -28,7 +28,7 @@ def test_in_distance_range():
     assert not eval_tracking.in_distance_range_pose(city_center, (0, 0), 0, 50)
 
 
-def test_get_pc_inside_box():
+def test_get_pc_inside_box() -> None:
     """
         test get_pc_inside_box(pc_raw,bbox) *bbox format is [p0,p1,p2,h]
 
@@ -56,45 +56,45 @@ def test_get_pc_inside_box():
         """
     bbox = np.array([np.array([[0], [0], [0]]), np.array([[2], [0], [0]]), np.array([[0], [5], [0]]), np.array(10)])
 
-    pc = ply_loader.load_ply(TEST_DATA_LOC / "1/lidar/PC_0.ply")
+    pc = ply_loader.load_ply(str(TEST_DATA_LOC / "1/lidar/PC_0.ply"))
 
     pc_inside = eval_utils.get_pc_inside_bbox(pc, bbox)
 
     assert len(pc_inside) == 3
 
 
-def test_leave_only_roi_region():
-    """
-        test leave_only_roi_region function
-        (lidar_pts,egovehicle_to_city_se3,ground_removal_method, city_name='MIA')
-        """
-    pc = ply_loader.load_ply(TEST_DATA_LOC / "1/lidar/PC_0.ply")
-    pose_data = read_json_file(TEST_DATA_LOC / "1/poses/city_SE3_egovehicle_0.json")
-    rotation = np.array(pose_data["rotation"])
-    translation = np.array(pose_data["translation"])
-    ego_R = quat2rotmat(rotation)
-    ego_t = translation
-    egovehicle_to_city_se3 = SE3(rotation=ego_R, translation=ego_t)
-    pts = eval_utils.leave_only_roi_region(pc, egovehicle_to_city_se3, ground_removal_method="map")
-    # might need toy data for map files
-
-
-def test_evaluation_track():
-    """
-        test eval_tracks function
-        """
-
-    # for testing, consider all point
-    eval_tracking.min_point_num = 0
-    centroid_methods = ["label_center", "average"]
-    temp_output = TEST_DATA_LOC / "tmp"
-    out_file = open(temp_output, "w+")
-
-    log = "1"
-    log_location = TEST_DATA_LOC / log
-
-    # sanity check, gt and results exactly the same
-    track_results_location = log_location / "per_sweep_annotations_amodal"
-    cm = centroid_methods[1]
-    eval_tracking.eval_tracks([track_results_location], [os.fspath(log_location)], D_MIN, D_MAX, out_file, cm)
-    out_file.close()
+# def test_leave_only_roi_region() -> None:
+#     """
+#         test leave_only_roi_region function
+#         (lidar_pts,egovehicle_to_city_se3,ground_removal_method, city_name='MIA')
+#         """
+#     pc = ply_loader.load_ply(TEST_DATA_LOC / "1/lidar/PC_0.ply")
+#     pose_data = read_json_file(TEST_DATA_LOC / "1/poses/city_SE3_egovehicle_0.json")
+#     rotation = np.array(pose_data["rotation"])
+#     translation = np.array(pose_data["translation"])
+#     ego_R = quat2rotmat(rotation)
+#     ego_t = translation
+#     egovehicle_to_city_se3 = SE3(rotation=ego_R, translation=ego_t)
+#     pts = eval_utils.leave_only_roi_region(pc, egovehicle_to_city_se3, ground_removal_method="map")
+#     # might need toy data for map files
+
+
+# def test_evaluation_track() -> None:
+#     """
+#         test eval_tracks function
+#         """
+
+#     # for testing, consider all point
+#     eval_tracking.min_point_num = 0
+#     centroid_methods = ["label_center", "average"]
+#     temp_output = TEST_DATA_LOC / "tmp"
+#     out_file = open(temp_output, "w+")
+
+#     log = "1"
+#     log_location = TEST_DATA_LOC / log
+
+#     # sanity check, gt and results exactly the same
+#     track_results_location = log_location / "per_sweep_annotations_amodal"
+#     cm = centroid_methods[1]
+#     eval_tracking.eval_tracks(track_results_location, os.fspath(log_location), D_MIN, D_MAX, out_file, cm)
+#     out_file.close()
diff --git a/tests/conftest.py b/tests/conftest.py
index 97f62b41..d115a4a2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -5,10 +5,11 @@
 import logging
 
 import pytest
+from _pytest.logging import LogCaptureFixture
 
 
-@pytest.fixture(autouse=True)
-def set_log_level(caplog):
+@pytest.fixture(autouse=True)  # type: ignore
+def set_log_level(caplog: LogCaptureFixture) -> None:
     """Set the log level.
 
     Set the log level to DEBUG for our testing to make sure that any bad log
diff --git a/tests/test_argoverse_forecasting_loader.py b/tests/test_argoverse_forecasting_loader.py
index e3328f76..1d5fc0c2 100644
--- a/tests/test_argoverse_forecasting_loader.py
+++ b/tests/test_argoverse_forecasting_loader.py
@@ -1,10 +1,10 @@
 # 
 """Forecasting Loader unit tests"""
 
-import glob
 import pathlib
 
 import numpy as np
+import pandas as pd
 import pytest
 
 from argoverse.data_loading.argoverse_forecasting_loader import ArgoverseForecastingLoader
@@ -12,17 +12,23 @@
 TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "forecasting"
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def data_loader() -> ArgoverseForecastingLoader:
     return ArgoverseForecastingLoader(TEST_DATA_LOC)
 
 
 def test_id_list(data_loader: ArgoverseForecastingLoader) -> None:
-    track_id_gt = [
-        "00000000-0000-0000-0000-000000000000",
-        "00000000-0000-0000-0000-000000007735",
-        "00000000-0000-0000-0000-000000008206",
-    ]
+    track_id_gt = (
+        pd.DataFrame(
+            [
+                "00000000-0000-0000-0000-000000000000",
+                "00000000-0000-0000-0000-000000007735",
+                "00000000-0000-0000-0000-000000008206",
+            ]
+        )
+        .values.flatten()
+        .tolist()
+    )
     assert data_loader.track_id_list == track_id_gt
 
 
diff --git a/tests/test_argoverse_tracking_loader.py b/tests/test_argoverse_tracking_loader.py
index 3adb52f6..72521fee 100644
--- a/tests/test_argoverse_tracking_loader.py
+++ b/tests/test_argoverse_tracking_loader.py
@@ -10,10 +10,10 @@
 from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader
 from argoverse.utils.camera_stats import CAMERA_LIST
 
-TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking"
+TEST_DATA_LOC = str(pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking")
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def data_loader() -> ArgoverseTrackingLoader:
     return ArgoverseTrackingLoader(TEST_DATA_LOC)
 
@@ -131,10 +131,14 @@ def test_calibration(data_loader: ArgoverseTrackingLoader) -> None:
 def test_pose(data_loader: ArgoverseTrackingLoader) -> None:
     for idx in range(len(data_loader.lidar_list)):
         pose = data_loader.get_pose(idx)
-        assert np.array_equal(
-            pose.inverse().transform_point_cloud(np.array([[0, 0, 0]])),
-            pose.inverse_transform_point_cloud(np.array([[0, 0, 0]])),
-        )
+
+        if pose is not None:
+            assert np.array_equal(
+                pose.inverse().transform_point_cloud(np.array([[0, 0, 0]])),
+                pose.inverse_transform_point_cloud(np.array([[0, 0, 0]])),
+            )
+        else:
+            assert False
 
 
 def test_idx_from_timestamp(data_loader: ArgoverseTrackingLoader) -> None:
diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index 3150aac0..b077d698 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -11,15 +11,22 @@
 from scipy.spatial.transform import Rotation as R
 
 from argoverse.data_loading.object_label_record import ObjectLabelRecord
-from argoverse.evaluation.detection_utils import AffFnType, DistFnType, compute_affinity_matrix, dist_fn, iou_aligned_3d
-from argoverse.evaluation.eval_detection import SIGNIFICANT_DIGITS, DetectionCfg, DetectionEvaluator
+from argoverse.evaluation.detection_utils import (
+    AffFnType,
+    DetectionCfg,
+    DistFnType,
+    compute_affinity_matrix,
+    dist_fn,
+    iou_aligned_3d,
+)
+from argoverse.evaluation.eval_detection import DetectionEvaluator
 from argoverse.utils.transform import quat_scipy2argo_vectorized
 
 TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection"
 logging.getLogger("matplotlib.font_manager").disabled = True
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def evaluator_identity() -> DetectionEvaluator:
     """Define an evaluator that compares a set of results to itself."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
@@ -28,7 +35,7 @@ def evaluator_identity() -> DetectionEvaluator:
     )
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def evaluator_assignment() -> DetectionEvaluator:
     """Define an evaluator that compares a set of results to one with an extra detection to check assignment."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
@@ -37,7 +44,7 @@ def evaluator_assignment() -> DetectionEvaluator:
     )
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def evaluator() -> DetectionEvaluator:
     """Definte an evaluator that compares a set of detections with known error to the ground truth."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
@@ -46,19 +53,19 @@ def evaluator() -> DetectionEvaluator:
     )
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def metrics_identity(evaluator_identity: DetectionEvaluator) -> DataFrame:
     """Get the metrics for an evaluator that compares a set of results to itself."""
     return evaluator_identity.evaluate()
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def metrics_assignment(evaluator_assignment: DetectionEvaluator) -> DataFrame:
     """Get the metrics for an evaluator that has extra detections to test for assignment errors."""
     return evaluator_assignment.evaluate()
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def metrics(evaluator: DetectionEvaluator) -> DataFrame:
     """Get the metrics for an evaluator with known error."""
     return evaluator.evaluate()
diff --git a/tests/test_frame_label_accumulator.py b/tests/test_frame_label_accumulator.py
index 57813465..5aa494cb 100644
--- a/tests/test_frame_label_accumulator.py
+++ b/tests/test_frame_label_accumulator.py
@@ -10,10 +10,10 @@
 
 from argoverse.data_loading.frame_label_accumulator import PerFrameLabelAccumulator
 
-TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking"
+TEST_DATA_LOC = str(pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking")
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def frame_acc() -> PerFrameLabelAccumulator:
     pfa = PerFrameLabelAccumulator(TEST_DATA_LOC, TEST_DATA_LOC, "test", save=False)
     pfa.accumulate_per_log_data()
@@ -23,6 +23,7 @@ def frame_acc() -> PerFrameLabelAccumulator:
 
 def test_traj_label_place_in_city(frame_acc: PerFrameLabelAccumulator) -> None:
     traj_list = frame_acc.get_log_trajectory_labels("1")
+    assert traj_list is not None
     city_frame_1_gt = [
         [[2.0, -1.0, -1.0], [2.0, -3.0, -1.0], [4.0, -1.0, -1.0], [4.0, -3.0, -1.0]],
         [[3.0, 1.0, 1.0], [3.0, 3.0, 1.0], [5.0, 1.0, 1.0], [5.0, 3.0, 1.0]],
diff --git a/tests/test_mayavi_utils_when_missing.py b/tests/test_mayavi_utils_when_missing.py
index 26108826..7fa3728d 100644
--- a/tests/test_mayavi_utils_when_missing.py
+++ b/tests/test_mayavi_utils_when_missing.py
@@ -21,7 +21,7 @@
 skip_if_mayavi = pytest.mark.skipif(not MISSING_MAYAVI, reason="mayavi installed")
 
 
-@skip_if_mayavi
+@skip_if_mayavi  # type: ignore
 def test_raises_when_we_try_to_use_a_missing_mayavi() -> None:
     n_mer, n_long = 6, 11
     pi = np.pi
diff --git a/tests/test_simple_track_dataloader.py b/tests/test_simple_track_dataloader.py
index d6a6bd0e..45e025c7 100644
--- a/tests/test_simple_track_dataloader.py
+++ b/tests/test_simple_track_dataloader.py
@@ -11,7 +11,7 @@
 _LOG_ID = "1"
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def data_loader() -> SimpleArgoverseTrackingDataLoader:
     return SimpleArgoverseTrackingDataLoader(os.fspath(_TEST_DATA), os.fspath(_TEST_DATA))
 
@@ -48,14 +48,18 @@ def test_get_labels_at_lidar_timestamp(data_loader: SimpleArgoverseTrackingDataL
 def test_get_closest_im_fpath_exists(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
     # Test data does have ring front cameras at timestamps 0,1,2,3. Compare with ground truth (gt)
     im_fpath = data_loader.get_closest_im_fpath(_LOG_ID, "ring_front_right", 2)
+    assert im_fpath is not None
+
     gt_im_fpath = f"test_data/tracking/{_LOG_ID}/ring_front_right/ring_front_right_2.jpg"
     assert "/".join(im_fpath.split("/")[-5:]) == gt_im_fpath
 
 
 def test_get_closest_lidar_fpath_found_match(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
     """ Just barely within 51 ms allowed buffer"""
-    cam_timestamp = 50 * 1e6
+    cam_timestamp = int(50 * 1e6)
     ply_fpath = data_loader.get_closest_lidar_fpath(_LOG_ID, cam_timestamp)
+
+    assert ply_fpath is not None
     gt_ply_fpath = f"test_data/tracking/{_LOG_ID}/lidar/PC_2.ply"
     assert "/".join(ply_fpath.split("/")[-5:]) == gt_ply_fpath
 
@@ -67,7 +71,7 @@ def test_get_closest_lidar_fpath_no_match(data_loader: SimpleArgoverseTrackingDa
     """
     max_allowed_interval = ((100 / 2) + 1) * 1e6
     log_max_lidar_timestamp = 2
-    cam_timestamp = log_max_lidar_timestamp + max_allowed_interval + 1
+    cam_timestamp = int(log_max_lidar_timestamp + max_allowed_interval + 1)
     ply_fpath = data_loader.get_closest_lidar_fpath(_LOG_ID, cam_timestamp)
     assert ply_fpath is None
 
diff --git a/tests/test_visualization_utils.py b/tests/test_visualization_utils.py
index 1abd9f33..ffe1a972 100644
--- a/tests/test_visualization_utils.py
+++ b/tests/test_visualization_utils.py
@@ -10,24 +10,24 @@
 from argoverse.data_loading.argoverse_tracking_loader import ArgoverseTrackingLoader
 from argoverse.visualization import visualization_utils
 
-TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking"
+TEST_DATA_LOC = str(pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "tracking")
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def axes() -> Iterator[plt.Axes]:
     fig = plt.gcf()
     yield plt.gca()
     plt.close(fig)
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def axes3d() -> Iterator[plt.Axes]:
     fig = plt.gcf()
     yield fig.add_subplot(111, projection="3d")
     plt.close(fig)
 
 
-@pytest.fixture
+@pytest.fixture  # type: ignore
 def data_loader() -> ArgoverseTrackingLoader:
     return ArgoverseTrackingLoader(TEST_DATA_LOC)
 
@@ -44,8 +44,8 @@ def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoad
     visualization_utils.draw_point_cloud_trajectory(axes, "title!", data_loader, 0, [1, 0])
 
 
-def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoader, axes3d: plt.Axes) -> None:
-    visualization_utils.draw_point_cloud_trajectory(axes3d, "title!", data_loader, 0)
+# def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoader, axes3d: plt.Axes) -> None:
+#     visualization_utils.draw_point_cloud_trajectory(axes3d, "title!", data_loader, 0)
 
 
 def test_make_grid_ring_camera_no_error(data_loader: ArgoverseTrackingLoader) -> None:

From dd99a5608878f387e0bbd4821197140af2cf76d0 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 11:40:19 -0400
Subject: [PATCH 089/113] More type fixes.

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5084dc03..4fa766e9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -23,5 +23,5 @@ repos:
     rev: d6e31ae
     hooks:
     - id: mypy
-      args: ["--ignore-missing", "--strict"]
+      args: ["--ignore-missing", "--strict", "--pretty"]
 exclude: "docs/.*$"

From ed51c79c92795ab77b58149c29cb0aceea200f04 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 11:59:04 -0400
Subject: [PATCH 090/113] Update travis build.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 99a21691..cb12d57e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ python:
     - "3.7"
 
 install:
-    - pip install tox-travis==0.12 tox==3.12.1 pre-commit==1.17.0
+    - pip install tox-travis tox pre-commit
 
 script:
     - travis_wait 45 tox

From d06e811b050c8b6bebb15a15944753aaf21b36b6 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 12:42:23 -0400
Subject: [PATCH 091/113] Attempt to fix environment issue.

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4fa766e9..60c67d80 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@ fail_fast: false
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: 0b70e28
+    rev: 5bd9e74
     hooks:
     - id: check-json
     - id: mixed-line-ending

From 3674c9392ba9e8eb7c739fe45ff9f5c89618ec3a Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 13:31:45 -0400
Subject: [PATCH 092/113] Updated pre-commit.

---
 .pre-commit-config.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 60c67d80..1fa9c079 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,25 +2,25 @@ fail_fast: false
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: 5bd9e74
+    rev: master
     hooks:
     - id: check-json
     - id: mixed-line-ending
       args: ['--fix=lf']
     - id: end-of-file-fixer
     - id: debug-statements
--   repo: https://github.com/pre-commit/mirrors-isort.git
-    rev: 687aecc
+-   repo: https://github.com/PyCQA/isort.git
+    rev: develop
     hooks:
     - id: isort
       args: ["-w", "120", "--profile", "black"]
 -   repo: https://github.com/python/black.git
-    rev: 1bbb01b
+    rev: master
     hooks:
     - id: black
       args: ["-l", "120"]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: d6e31ae
+    rev: master
     hooks:
     - id: mypy
       args: ["--ignore-missing", "--strict", "--pretty"]

From db8baa619a1525c08738b6097d9bb09fb8e51c24 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 13:33:54 -0400
Subject: [PATCH 093/113] Updated pre-commit.

---
 .pre-commit-config.yaml | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1fa9c079..cf60c0a3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,24 +4,26 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
     rev: master
     hooks:
-    - id: check-json
-    - id: mixed-line-ending
-      args: ['--fix=lf']
-    - id: end-of-file-fixer
-    - id: debug-statements
+      - id: check-json
+      - id: mixed-line-ending
+        args: ['--fix=lf']
+      - id: end-of-file-fixer
+      - id: debug-statements
 -   repo: https://github.com/PyCQA/isort.git
     rev: develop
     hooks:
-    - id: isort
-      args: ["-w", "120", "--profile", "black"]
+      - id: isort
+        args: ["-w", "120", "--profile", "black"]
 -   repo: https://github.com/python/black.git
     rev: master
     hooks:
-    - id: black
-      args: ["-l", "120"]
+      - id: black
+        args: ["-l", "120"]
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: master
     hooks:
-    - id: mypy
-      args: ["--ignore-missing", "--strict", "--pretty"]
+      - id: mypy
+        args: ["--ignore-missing", "--strict", "--pretty"]
+        additional_dependencies:
+        - 'virtualenv'
 exclude: "docs/.*$"

From 1e4847d2d49793a113bd1472e577f0c209fc566d Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 14:57:19 -0400
Subject: [PATCH 094/113] Remove mypy from pre-commit.

---
 .pre-commit-config.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index cf60c0a3..a9976b00 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -19,11 +19,4 @@ repos:
     hooks:
       - id: black
         args: ["-l", "120"]
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: master
-    hooks:
-      - id: mypy
-        args: ["--ignore-missing", "--strict", "--pretty"]
-        additional_dependencies:
-        - 'virtualenv'
 exclude: "docs/.*$"

From 5b5b9d23d6352f10cd8450f524cef1aace0514dd Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 15:30:30 -0400
Subject: [PATCH 095/113] Revert back to previous pre-commit.

---
 .pre-commit-config.yaml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a9976b00..0a11eae3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,21 +2,21 @@ fail_fast: false
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: master
+    rev: 0b70e28
     hooks:
-      - id: check-json
-      - id: mixed-line-ending
-        args: ['--fix=lf']
-      - id: end-of-file-fixer
-      - id: debug-statements
--   repo: https://github.com/PyCQA/isort.git
-    rev: develop
+    - id: check-json
+    - id: mixed-line-ending
+      args: ['--fix=lf']
+    - id: end-of-file-fixer
+    - id: debug-statements
+-   repo: https://github.com/pre-commit/mirrors-isort.git
+    rev: dc9ed97
     hooks:
-      - id: isort
-        args: ["-w", "120", "--profile", "black"]
+    - id: isort
+      args: ["-w", "120", "--multi-line=3", "--trailing-comma", "--force-grid-wrap=0", "--use-parentheses"]
 -   repo: https://github.com/python/black.git
-    rev: master
+    rev: 1bbb01b
     hooks:
-      - id: black
-        args: ["-l", "120"]
+    - id: black
+      args: ["-l", "120"]
 exclude: "docs/.*$"

From 9afdf8f28d8f094e07b1faaefd0d563e6bec65cb Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 16:41:03 -0400
Subject: [PATCH 096/113] Reverting travis configuration as well.

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index cb12d57e..3c2f93af 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,10 +4,10 @@ python:
     - "3.7"
 
 install:
-    - pip install tox-travis tox pre-commit
+    - pip install tox-travis==0.12 tox==3.12.1 pre-commit==1.17.0
 
 script:
-    - travis_wait 45 tox
+    - tox
     - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then pre-commit run --show-diff-on-failure --origin $TRAVIS_COMMIT --source $TRAVIS_BRANCH; fi'
 
 notifications:

From 3609cf61c6e335f9421228b180598419fbdb1f2e Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 17:13:07 -0400
Subject: [PATCH 097/113] Docstring update.

---
 argoverse/evaluation/eval_detection.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index 80e33dbe..b9fb6fd4 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -30,9 +30,9 @@
             is computed as the product of mAP with the sum of the complements of the true positive
             errors (after normalization), i.e.,
 
-            Average Translation Measure (ATM): ATE / TP_THRESHOLD; 0 <= 1 - ATE / TP_THRESHOLD <= 1
-            Average Scaling Measure (ASM): 1 - ASE / 1;  0 <= 1 - ASE / 1 <= 1
-            Average Orientation Measure (AOM): 1 - AOE / PI; 0 <= 1 - AOE / PI <= 1
+            - Average Translation Measure (ATM): ATE / TP_THRESHOLD; 0 <= 1 - ATE / TP_THRESHOLD <= 1
+            - Average Scaling Measure (ASM): 1 - ASE / 1;  0 <= 1 - ASE / 1 <= 1
+            - Average Orientation Measure (AOM): 1 - AOE / PI; 0 <= 1 - AOE / PI <= 1
 
             These (as well as AP) are averaged over each detection class to produce:
 

From 1754a0c2b07949a4ffd30d45a8a885fac4900ea0 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 17:13:11 -0400
Subject: [PATCH 098/113] Docstring update.

---
 argoverse/evaluation/eval_detection.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index b9fb6fd4..c3ab0694 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -30,13 +30,16 @@
             is computed as the product of mAP with the sum of the complements of the true positive
             errors (after normalization), i.e.,
 
-            - Average Translation Measure (ATM): ATE / TP_THRESHOLD; 0 <= 1 - ATE / TP_THRESHOLD <= 1
-            - Average Scaling Measure (ASM): 1 - ASE / 1;  0 <= 1 - ASE / 1 <= 1
-            - Average Orientation Measure (AOM): 1 - AOE / PI; 0 <= 1 - AOE / PI <= 1
+            - Average Translation Measure (ATM): ATE / TP_THRESHOLD; 0 <= 1 - ATE / TP_THRESHOLD <= 1.
+            - Average Scaling Measure (ASM): 1 - ASE / 1;  0 <= 1 - ASE / 1 <= 1.
+            - Average Orientation Measure (AOM): 1 - AOE / PI; 0 <= 1 - AOE / PI <= 1.
 
             These (as well as AP) are averaged over each detection class to produce:
 
-            mAP, mATM, mASM, mAOM.
+            - mAP
+            - mATM
+            - mASM
+            - mAOM
 
             Lastly, the Composite Detection Score is computed as:
 

From 16d49ab15a52bbd8896cc75d80d97c4a8ce447ab Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 17:17:57 -0400
Subject: [PATCH 099/113] Add back tox timeout.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3c2f93af..1171a969 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ install:
     - pip install tox-travis==0.12 tox==3.12.1 pre-commit==1.17.0
 
 script:
-    - tox
+    - travis_wait 30 tox
     - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then pre-commit run --show-diff-on-failure --origin $TRAVIS_COMMIT --source $TRAVIS_BRANCH; fi'
 
 notifications:

From ad1619cbd8aacb6694680ffcd6c6569518eb91a6 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 18:33:48 -0400
Subject: [PATCH 100/113] Modified isort.

---
 .pre-commit-config.yaml                | 4 ++--
 argoverse/evaluation/eval_detection.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0a11eae3..1d358b99 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,10 +10,10 @@ repos:
     - id: end-of-file-fixer
     - id: debug-statements
 -   repo: https://github.com/pre-commit/mirrors-isort.git
-    rev: dc9ed97
+    rev: 687aecc
     hooks:
     - id: isort
-      args: ["-w", "120", "--multi-line=3", "--trailing-comma", "--force-grid-wrap=0", "--use-parentheses"]
+      args: ["--profile", "black"]
 -   repo: https://github.com/python/black.git
     rev: 1bbb01b
     hooks:
diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index c3ab0694..b6a564b2 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -41,6 +41,7 @@
             - mASM
             - mAOM
 
+
             Lastly, the Composite Detection Score is computed as:
 
             CDS = mAP * (mATE + mASE + mAOE); 0 <= mAP * (mATE + mASE + mAOE) <= 1.

From 12f2056506fb3383a4bbcc84a17735386ae2a378 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 18:59:26 -0400
Subject: [PATCH 101/113] Another style test.

---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1d358b99..d4bad340 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,8 +14,8 @@ repos:
     hooks:
     - id: isort
       args: ["--profile", "black"]
--   repo: https://github.com/python/black.git
-    rev: 1bbb01b
+-   repo: https://github.com/psf/black
+    rev: 172c0a7
     hooks:
     - id: black
       args: ["-l", "120"]

From 467304357e8393432a0ae99b5986da4c3e733b3f Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 19:03:35 -0400
Subject: [PATCH 102/113] Black update.

---
 .../data_loading/frame_label_accumulator.py   | 15 +--
 argoverse/data_loading/frame_record.py        | 22 ++---
 argoverse/data_loading/object_label_record.py | 18 ++--
 argoverse/data_loading/pose_loader.py         | 12 +--
 .../data_loading/simple_track_dataloader.py   |  2 +-
 .../data_loading/synchronization_database.py  | 83 ++++++++--------
 argoverse/data_loading/vector_map_loader.py   | 24 +++--
 argoverse/evaluation/detection_utils.py       |  2 +-
 argoverse/map_representation/map_api.py       |  7 +-
 argoverse/utils/calibration.py                |  6 +-
 argoverse/utils/camera_stats.py               |  2 +-
 argoverse/utils/forecasting_evaluation.py     | 10 +-
 argoverse/utils/interpolate.py                |  2 +-
 argoverse/utils/line_projection.py            |  4 +-
 argoverse/utils/make_att_files.py             | 18 ++--
 argoverse/utils/manhattan_search.py           | 10 +-
 .../visualize_30hz_benchmark_data_on_map.py   |  5 +-
 integration_tests/test_map_api.py             | 24 ++---
 integration_tests/test_mayavi_utils.py        |  7 +-
 .../test_plane_visualization_utils.py         |  9 +-
 integration_tests/test_tracker_eval.py        | 52 +++++-----
 tests/test_bfs.py                             |  8 +-
 tests/test_centerline_utils.py                | 24 ++---
 tests/test_cuboid_interior.py                 |  3 +-
 tests/test_cv2_plotting_utils.py              | 24 ++---
 tests/test_eval_tracking.py                   | 96 +++++++++----------
 tests/test_ffmpeg_utils_unit.py               |  6 +-
 tests/test_geometry.py                        | 26 ++---
 tests/test_interpolate.py                     | 53 +++++-----
 tests/test_mpl_plotting_utils.py              | 16 +---
 tests/test_simple_track_dataloader.py         | 10 +-
 tests/test_subprocess_utils.py                | 12 +--
 32 files changed, 309 insertions(+), 303 deletions(-)

diff --git a/argoverse/data_loading/frame_label_accumulator.py b/argoverse/data_loading/frame_label_accumulator.py
index e6430e3e..b72bfc98 100644
--- a/argoverse/data_loading/frame_label_accumulator.py
+++ b/argoverse/data_loading/frame_label_accumulator.py
@@ -16,7 +16,10 @@
 from argoverse.data_loading.object_label_record import ObjectLabelRecord
 from argoverse.data_loading.pose_loader import get_city_SE3_egovehicle_at_sensor_t
 from argoverse.data_loading.synchronization_database import SynchronizationDB
-from argoverse.data_loading.trajectory_loader import TrajectoryLabel, load_json_track_labels
+from argoverse.data_loading.trajectory_loader import (
+    TrajectoryLabel,
+    load_json_track_labels,
+)
 from argoverse.utils.json_utils import read_json_file
 from argoverse.utils.pkl_utils import load_pkl_dictionary, save_pkl_dictionary
 from argoverse.utils.se3 import SE3
@@ -26,7 +29,7 @@
 
 
 class PerFrameLabelAccumulator:
-    """ We will cache the accumulated track label trajectories per city, per log, and per frame.
+    """We will cache the accumulated track label trajectories per city, per log, and per frame.
     In order to plot each frame sequentially, one at a time, we need to aggregate beforehand
     the tracks and cuboids for each frame.
 
@@ -43,7 +46,7 @@ class PerFrameLabelAccumulator:
     def __init__(
         self, dataset_dir: str, labels_dir: str, experiment_prefix: str, bboxes_3d: bool = False, save: bool = True
     ) -> None:
-        """ Initialize PerFrameLabelAccumulator object for use with tracking benchmark data.
+        """Initialize PerFrameLabelAccumulator object for use with tracking benchmark data.
 
         Args:
             dataset_dir (str): Dataset directory.
@@ -126,7 +129,7 @@ def accumulate_per_log_data(self, log_id: Optional[str] = None) -> None:
         logger.info(f"Miami has {MIAMI_CUBOID_COUNT} and Pittsburgh has {PITT_CUBOID_COUNT} cuboids")
 
     def get_log_trajectory_labels(self, log_id: str) -> Optional[List[TrajectoryLabel]]:
-        """ Create a very large list with all of the trajectory data.
+        """Create a very large list with all of the trajectory data.
 
         Treat a single object cuboid label as one step in a trajectory.
         Then we can share the same representation for both.
@@ -144,7 +147,7 @@ def get_log_trajectory_labels(self, log_id: str) -> Optional[List[TrajectoryLabe
             return None
 
     def place_trajectory_in_city_frame(self, traj_label: TrajectoryLabel, log_id: str) -> np.ndarray:
-        """ Place trajectory in the city frame
+        """Place trajectory in the city frame
         Args:
             traj_label (TrajectoryLabel): instance of the TrajectoryLabel class.
             log_id (str): Log id.
@@ -209,7 +212,7 @@ def place_trajectory_in_city_frame(self, traj_label: TrajectoryLabel, log_id: st
     def convert_bbox_to_city_frame(
         self, lidar_timestamp_ns: int, dataset_dir: str, log_id: str, bbox_ego_frame: np.ndarray
     ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:
-        """ Convert bounding box to city frame.
+        """Convert bounding box to city frame.
         Args:
             lidar_timestamp_ns (int): Lidar timestamp.
             dataset_dir (str): representing full path to the log_ids.
diff --git a/argoverse/data_loading/frame_record.py b/argoverse/data_loading/frame_record.py
index d6edb5bc..d57dd3df 100644
--- a/argoverse/data_loading/frame_record.py
+++ b/argoverse/data_loading/frame_record.py
@@ -9,9 +9,9 @@
 
 class FrameRecord:
     """
-        Store representation of a bounding box in some timeframe, in different coordinate systems.
-        This bounding box comes from a track that shares the same color.
-        """
+    Store representation of a bounding box in some timeframe, in different coordinate systems.
+    This bounding box comes from a track that shares the same color.
+    """
 
     def __init__(
         self,
@@ -22,14 +22,14 @@ def __init__(
         track_uuid: str,
         obj_class_str: str,
     ) -> None:
-        """ Initialize FrameRecord.
-            Args:
-              bbox_city_fr: bounding box for city frame.
-              bbox_ego_frame: bounding box for ego frame.
-              occlusion_val: occlusion value.
-              color: tuple representing color. RGB values should be within [0,1] range.
-              track_uuid: track uuid
-              obj_class_str: object class string
+        """Initialize FrameRecord.
+        Args:
+          bbox_city_fr: bounding box for city frame.
+          bbox_ego_frame: bounding box for ego frame.
+          occlusion_val: occlusion value.
+          color: tuple representing color. RGB values should be within [0,1] range.
+          track_uuid: track uuid
+          obj_class_str: object class string
         """
         self.bbox_city_fr = bbox_city_fr
         self.bbox_ego_frame = bbox_ego_frame
diff --git a/argoverse/data_loading/object_label_record.py b/argoverse/data_loading/object_label_record.py
index ec6eb3ed..db8d06ba 100644
--- a/argoverse/data_loading/object_label_record.py
+++ b/argoverse/data_loading/object_label_record.py
@@ -190,18 +190,18 @@ def draw_rect(selected_corners: np.array, color: Tuple[int, int, int]) -> None:
 def form_obj_label_from_json(label: Dict[str, Any]) -> Tuple[np.array, str]:
     """Construct object from loaded json.
 
-    The dictionary loaded from saved json file is expected to have the
-    following fields::
+     The dictionary loaded from saved json file is expected to have the
+     following fields::
 
-        ['frame_index', 'center', 'rotation', 'length', 'width', 'height',
-        'track_label_uuid', 'occlusion', 'on_driveable_surface', 'key_frame',
-        'stationary', 'label_class']
+         ['frame_index', 'center', 'rotation', 'length', 'width', 'height',
+         'track_label_uuid', 'occlusion', 'on_driveable_surface', 'key_frame',
+         'stationary', 'label_class']
 
-   Args:
-        label: Python dictionary that was loaded from saved json file
+    Args:
+         label: Python dictionary that was loaded from saved json file
 
-    Returns:
-        Tuple of (bbox_ego_frame, color); bbox is a numpy array of shape (4,3); color is "g" or "r"
+     Returns:
+         Tuple of (bbox_ego_frame, color); bbox is a numpy array of shape (4,3); color is "g" or "r"
     """
     tr_x = label["center"]["x"]
     tr_y = label["center"]["y"]
diff --git a/argoverse/data_loading/pose_loader.py b/argoverse/data_loading/pose_loader.py
index b955b70b..805adaa3 100644
--- a/argoverse/data_loading/pose_loader.py
+++ b/argoverse/data_loading/pose_loader.py
@@ -16,13 +16,13 @@
 def get_city_SE3_egovehicle_at_sensor_t(sensor_timestamp: int, dataset_dir: str, log_id: str) -> Optional[SE3]:
     """Get transformation from ego-vehicle to city coordinates at a given timestamp.
 
-        Args:
-            sensor_timestamp: integer representing timestamp when sensor measurement captured, in nanoseconds
-            dataset_dir:
-            log_id: string representing unique log identifier
+    Args:
+        sensor_timestamp: integer representing timestamp when sensor measurement captured, in nanoseconds
+        dataset_dir:
+        log_id: string representing unique log identifier
 
-        Returns:
-            SE3 for translating ego-vehicle coordinates to city coordinates if found, else None.
+    Returns:
+        SE3 for translating ego-vehicle coordinates to city coordinates if found, else None.
     """
     pose_fpath = f"{dataset_dir}/{log_id}/poses/city_SE3_egovehicle_{sensor_timestamp}.json"
     if not Path(pose_fpath).exists():
diff --git a/argoverse/data_loading/simple_track_dataloader.py b/argoverse/data_loading/simple_track_dataloader.py
index 0d0c69b0..270b3600 100644
--- a/argoverse/data_loading/simple_track_dataloader.py
+++ b/argoverse/data_loading/simple_track_dataloader.py
@@ -114,7 +114,7 @@ def get_ordered_log_ply_fpaths(self, log_id: str) -> List[str]:
             log_id: str, unique ID of vehicle log
         Returns:
             ply_fpaths: List of strings, representing paths to ply files in this log
-            """
+        """
         ply_fpaths = sorted(glob.glob(f"{self.data_dir}/{log_id}/lidar/PC_*.ply"))
         return ply_fpaths
 
diff --git a/argoverse/data_loading/synchronization_database.py b/argoverse/data_loading/synchronization_database.py
index 83fd88e5..25be9ba5 100644
--- a/argoverse/data_loading/synchronization_database.py
+++ b/argoverse/data_loading/synchronization_database.py
@@ -16,14 +16,14 @@
 
 
 def get_timestamps_from_sensor_folder(sensor_folder_wildcard: str) -> np.ndarray:
-    """ Timestamp always lies at end of filename
+    """Timestamp always lies at end of filename
 
-        Args:
-            sensor_folder_wildcard: string to glob to find all filepaths for a particular
-                        sensor files within a single log run
+    Args:
+        sensor_folder_wildcard: string to glob to find all filepaths for a particular
+                    sensor files within a single log run
 
-        Returns:
-            Numpy array of integers, representing timestamps
+    Returns:
+        Numpy array of integers, representing timestamps
     """
 
     path_generator = glob.glob(sensor_folder_wildcard)
@@ -34,22 +34,22 @@ def get_timestamps_from_sensor_folder(sensor_folder_wildcard: str) -> np.ndarray
 
 def find_closest_integer_in_ref_arr(query_int: int, ref_arr: np.ndarray) -> Tuple[int, int]:
     """
-        Find the closest integer to any integer inside a reference array, and the corresponding
-        difference.
+    Find the closest integer to any integer inside a reference array, and the corresponding
+    difference.
 
-        In our use case, the query integer represents a nanosecond-discretized timestamp, and the
-        reference array represents a numpy array of nanosecond-discretized timestamps.
+    In our use case, the query integer represents a nanosecond-discretized timestamp, and the
+    reference array represents a numpy array of nanosecond-discretized timestamps.
 
-        Instead of sorting the whole array of timestamp differences, we just
-        take the minimum value (to speed up this function).
+    Instead of sorting the whole array of timestamp differences, we just
+    take the minimum value (to speed up this function).
 
-        Args:
-            query_int: query integer,
-            ref_arr: Numpy array of integers
+    Args:
+        query_int: query integer,
+        ref_arr: Numpy array of integers
 
-        Returns:
-            integer, representing the closest integer found in a reference array to a query
-            integer, representing the integer difference between the match and query integers
+    Returns:
+        integer, representing the closest integer found in a reference array to a query
+        integer, representing the integer difference between the match and query integers
     """
     closest_ind = np.argmin(np.absolute(ref_arr - query_int))
     closest_int = cast(int, ref_arr[closest_ind])  # mypy does not understand numpy arrays
@@ -76,15 +76,15 @@ class SynchronizationDB:
     MAX_LIDAR_ANYCAM_TIMESTAMP_DIFF = 102 * (1.0 / 2) * (1.0 / 1000) * 1e9
 
     def __init__(self, dataset_dir: str, collect_single_log_id: Optional[str] = None) -> None:
-        """ Build the SynchronizationDB.
-            Note that the timestamps for each camera channel are not identical, but they are clustered together.
+        """Build the SynchronizationDB.
+        Note that the timestamps for each camera channel are not identical, but they are clustered together.
 
-            Args:
-                dataset_dir: path to dataset.
-                collect_single_log_id: log id to process. (All if not set)
+        Args:
+            dataset_dir: path to dataset.
+            collect_single_log_id: log id to process. (All if not set)
 
-            Returns:
-                None
+        Returns:
+            None
         """
         logger.info("Building SynchronizationDB")
 
@@ -112,20 +112,19 @@ def __init__(self, dataset_dir: str, collect_single_log_id: Optional[str] = None
             self.per_log_lidartimestamps_index[log_id] = lidar_timestamps
 
     def get_valid_logs(self) -> Iterable[str]:
-        """ Return the log_ids for which the SynchronizationDatabase contains pose information.
-        """
+        """Return the log_ids for which the SynchronizationDatabase contains pose information."""
         return self.per_log_camtimestamps_index.keys()
 
     def get_closest_lidar_timestamp(self, cam_timestamp: int, log_id: str) -> Optional[int]:
-        """ Given an image timestamp, find the synchronized corresponding LiDAR timestamp.
-            This LiDAR timestamp should have the closest absolute timestamp to the image timestamp.
+        """Given an image timestamp, find the synchronized corresponding LiDAR timestamp.
+        This LiDAR timestamp should have the closest absolute timestamp to the image timestamp.
 
-            Args:
-                cam_timestamp: integer
-                log_id: string
+        Args:
+            cam_timestamp: integer
+            log_id: string
 
-            Returns:
-                closest_lidar_timestamp: closest timestamp
+        Returns:
+            closest_lidar_timestamp: closest timestamp
         """
         if log_id not in self.per_log_lidartimestamps_index:
             return None
@@ -147,16 +146,16 @@ def get_closest_lidar_timestamp(self, cam_timestamp: int, log_id: str) -> Option
         return closest_lidar_timestamp
 
     def get_closest_cam_channel_timestamp(self, lidar_timestamp: int, camera_name: str, log_id: str) -> Optional[int]:
-        """ Given a LiDAR timestamp, find the synchronized corresponding image timestamp for a particular camera.
-            This image timestamp should have the closest absolute timestamp.
+        """Given a LiDAR timestamp, find the synchronized corresponding image timestamp for a particular camera.
+        This image timestamp should have the closest absolute timestamp.
 
-            Args:
-                lidar_timestamp: integer
-                camera_name: string, representing path to log directories
-                log_id: string
+        Args:
+            lidar_timestamp: integer
+            camera_name: string, representing path to log directories
+            log_id: string
 
-            Returns:
-                closest_cam_ch_timestamp: closest timestamp
+        Returns:
+            closest_cam_ch_timestamp: closest timestamp
         """
         if (
             log_id not in self.per_log_camtimestamps_index
diff --git a/argoverse/data_loading/vector_map_loader.py b/argoverse/data_loading/vector_map_loader.py
index 30aa21ff..4a5ab028 100644
--- a/argoverse/data_loading/vector_map_loader.py
+++ b/argoverse/data_loading/vector_map_loader.py
@@ -44,7 +44,17 @@
 import pathlib
 import sys
 import xml.etree.ElementTree as ET
-from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union, cast
+from typing import (
+    Any,
+    Dict,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 
 import numpy as np
 
@@ -64,13 +74,13 @@ class Node:
 
     def __init__(self, id: int, x: float, y: float, height: Optional[float] = None):
         """
-            Args:
-                id: representing unique node ID
-                x: x-coordinate in city reference system
-                y: y-coordinate in city reference system
+        Args:
+            id: representing unique node ID
+            x: x-coordinate in city reference system
+            y: y-coordinate in city reference system
 
-            Returns:
-                None
+        Returns:
+            None
         """
         self.id = id
         self.x = x
diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index 4a3942a2..e4b43a4b 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -368,7 +368,7 @@ def iou_aligned_3d(dt_dims: pd.DataFrame, gt_dims: pd.DataFrame) -> np.ndarray:
     Args:
         dt_dims: Detections (N, 3).
         gt_dims: Ground truth labels (N, 3).
-    
+
     Returns:
         Intersection-over-union between the detections and their assigned ground
         truth labels (N,).
diff --git a/argoverse/map_representation/map_api.py b/argoverse/map_representation/map_api.py
index 2a28a8d2..759c2443 100644
--- a/argoverse/map_representation/map_api.py
+++ b/argoverse/map_representation/map_api.py
@@ -25,7 +25,10 @@
     find_all_polygon_bboxes_overlapping_query_bbox,
     find_local_polygons,
 )
-from argoverse.utils.mpl_plotting_utils import plot_lane_segment_patch, visualize_centerline
+from argoverse.utils.mpl_plotting_utils import (
+    plot_lane_segment_patch,
+    visualize_centerline,
+)
 from argoverse.utils.pkl_utils import load_pkl_dictionary
 from argoverse.utils.se2 import SE2
 
@@ -790,7 +793,7 @@ def get_cl_from_lane_seq(self, lane_seqs: Iterable[List[int]], city_name: str) -
     def get_candidate_centerlines_for_traj(
         self, xy: np.ndarray, city_name: str, viz: bool = False, max_search_radius: float = 50.0
     ) -> List[np.ndarray]:
-        """ Get centerline candidates upto a threshold. .
+        """Get centerline candidates upto a threshold. .
 
         Algorithm:
         1. Take the lanes in the bubble of last obs coordinate
diff --git a/argoverse/utils/calibration.py b/argoverse/utils/calibration.py
index bc10218c..93d018bb 100644
--- a/argoverse/utils/calibration.py
+++ b/argoverse/utils/calibration.py
@@ -155,7 +155,7 @@ def project_cam_to_ego(self, pts_3d_rect: np.array) -> np.ndarray:
         return np.linalg.inv((self.extrinsic)).dot(self.cart2hom(pts_3d_rect).transpose()).transpose()[:, 0:3]
 
     def project_image_to_ego(self, uv_depth: np.array) -> np.ndarray:
-        """ Project 2D image with depth to egovehicle coordinate.
+        """Project 2D image with depth to egovehicle coordinate.
 
         Args:
             uv_depth: nx3 first two channels are uv, 3rd channel
@@ -168,7 +168,7 @@ def project_image_to_ego(self, uv_depth: np.array) -> np.ndarray:
         return self.project_cam_to_ego(uv_cam)
 
     def project_image_to_cam(self, uv_depth: np.array) -> np.ndarray:
-        """ Project 2D image with depth to camera coordinate.
+        """Project 2D image with depth to camera coordinate.
 
         Args:
             uv_depth: nx3 first two channels are uv, 3rd channel
@@ -217,7 +217,7 @@ def load_image(img_filename: Union[str, Path]) -> np.ndarray:
 
 
 def load_calib(calib_filepath: Union[str, Path]) -> Dict[Any, Calibration]:
-    """ Load Calibration object for all camera from calibration filepath
+    """Load Calibration object for all camera from calibration filepath
 
     Args:
         calib_filepath (str): path to the calibration file
diff --git a/argoverse/utils/camera_stats.py b/argoverse/utils/camera_stats.py
index 1d9aab7e..5bfcf082 100644
--- a/argoverse/utils/camera_stats.py
+++ b/argoverse/utils/camera_stats.py
@@ -33,7 +33,7 @@
 
 
 def get_image_dims_for_camera(camera_name: str) -> Tuple[Optional[int], Optional[int]]:
-    """ Get image dimensions for camera.
+    """Get image dimensions for camera.
     Args:
         camera_name: Camera name.
 
diff --git a/argoverse/utils/forecasting_evaluation.py b/argoverse/utils/forecasting_evaluation.py
index 2a1f759e..d9995456 100644
--- a/argoverse/utils/forecasting_evaluation.py
+++ b/argoverse/utils/forecasting_evaluation.py
@@ -18,12 +18,12 @@ def compute_summed_distance_point_cloud2D(points_a: np.ndarray, points_b: np.nda
 def evaluate_prediction(
     pred_traj: np.ndarray, ground_truth_traj: np.ndarray, eval_method: str = "EVAL_DESTINATION_ONLY"
 ) -> np.ndarray:
-    """ Compute the error as L2 norm in trajectories
+    """Compute the error as L2 norm in trajectories
 
-        Args:
-            pred_traj: numpy n-d array with dims (N x 2)
-            ground_truth_traj: numpy n-d array with dims (N x 2)
-            eval_method:
+    Args:
+        pred_traj: numpy n-d array with dims (N x 2)
+        ground_truth_traj: numpy n-d array with dims (N x 2)
+        eval_method:
     """
     if eval_method == "EVAL_DESTINATION_ONLY":
         return np.linalg.norm(pred_traj[-1] - ground_truth_traj[-1])
diff --git a/argoverse/utils/interpolate.py b/argoverse/utils/interpolate.py
index f2eede2b..72b22985 100644
--- a/argoverse/utils/interpolate.py
+++ b/argoverse/utils/interpolate.py
@@ -155,7 +155,7 @@ def eliminate_duplicates_2d(px: np.ndarray, py: np.ndarray) -> Tuple[np.ndarray,
 
 
 def interp_arc(t: int, px: np.ndarray, py: np.ndarray) -> np.ndarray:
-    """ Linearly interpolate equally-spaced points along a polyline.
+    """Linearly interpolate equally-spaced points along a polyline.
 
     We use a chordal parameterization so that interpolated arc-lengths
     will approximate original polyline chord lengths.
diff --git a/argoverse/utils/line_projection.py b/argoverse/utils/line_projection.py
index 1e3ddb6c..06552ce9 100644
--- a/argoverse/utils/line_projection.py
+++ b/argoverse/utils/line_projection.py
@@ -11,7 +11,7 @@
 def project_to_line_seq(
     trajectory: np.ndarray, lines: Sequence[np.ndarray], interpolate_more: bool = True
 ) -> Tuple[float, np.ndarray]:
-    """ Project a trajectory onto a line sequence.
+    """Project a trajectory onto a line sequence.
 
     Args:
         trajectory: An array of shape (N,2) that we will project onto a polyline.
@@ -35,7 +35,7 @@ def project_to_line_seq(
 def project_to_line(
     trajectory: np.ndarray, center_polyline: np.ndarray, enforce_same_density: bool = False
 ) -> Tuple[float, np.ndarray]:
-    """ Project a trajectory onto a polyline.
+    """Project a trajectory onto a polyline.
 
     Args:
         trajectory: An array of shape (N,2) that we will project onto a polyline.
diff --git a/argoverse/utils/make_att_files.py b/argoverse/utils/make_att_files.py
index 7f3d32be..e8295c28 100644
--- a/argoverse/utils/make_att_files.py
+++ b/argoverse/utils/make_att_files.py
@@ -50,7 +50,7 @@ def save_bev_img(
     pc: np.ndarray,
 ) -> None:
     """
-    Plot results on bev images and save 
+    Plot results on bev images and save
     """
     image_size = 2000
     image_scale = 10
@@ -123,14 +123,14 @@ def save_bev_img(
 
 
 def bspline_1d(x: np.array, y: np.array, s: float = 20.0, k: int = 3) -> np.array:
-    """ Perform B-Spline smoothing of trajectories for temporal noise reduction
-    
+    """Perform B-Spline smoothing of trajectories for temporal noise reduction
+
     Args:
         x: N-length np array
         y: N-length np array
         s: smoothing condition
         k: degree of the spline fit
-        
+
     Returns:
         smoothed trajectory
     """
@@ -144,11 +144,11 @@ def bspline_1d(x: np.array, y: np.array, s: float = 20.0, k: int = 3) -> np.arra
 
 
 def derivative(x: np.array) -> np.array:
-    """ Compute time derivatives for velocity and acceleration
-    
+    """Compute time derivatives for velocity and acceleration
+
     Args:
         x: N-length Numpy array, with indices at consecutive timestamps
-    
+
     Returns:
         dx/dt: N-length Numpy array, with derivative of x w.r.t. timestep
     """
@@ -162,10 +162,10 @@ def derivative(x: np.array) -> np.array:
 def compute_v_a(traj: np.array) -> Tuple[np.array, np.array]:
     """
     Compute velocity and acceleration
-    
+
     Args:
         traj: Numpy array of shape Nx3 representing 3-d trajectory
-    
+
     Returns:
         velocity: Numpy array representing (d traj)/dt
         acceleration: Numpy array representing (d velocity)/dt
diff --git a/argoverse/utils/manhattan_search.py b/argoverse/utils/manhattan_search.py
index 0be5eeae..b363451d 100644
--- a/argoverse/utils/manhattan_search.py
+++ b/argoverse/utils/manhattan_search.py
@@ -11,7 +11,7 @@
 
 
 def compute_polygon_bboxes(polygons: np.ndarray) -> np.ndarray:
-    """ Compute the minimum size enclosing xy bounding box for each polygon that is provided as input.
+    """Compute the minimum size enclosing xy bounding box for each polygon that is provided as input.
     Args:
         polygons: an array of type 'O' (object) with shape (n,). Each object has shape (m, 3+).
 
@@ -29,7 +29,7 @@ def compute_polygon_bboxes(polygons: np.ndarray) -> np.ndarray:
 
 
 def compute_point_cloud_bbox(point_cloud: np.ndarray, verbose: bool = False) -> np.ndarray:
-    """ Given a set of 2D or 3D points, find the minimum size axis-aligned bounding box in the xy plane (ground plane).
+    """Given a set of 2D or 3D points, find the minimum size axis-aligned bounding box in the xy plane (ground plane).
 
     Args:
         point_cloud: an array of dim (N,3) or (N,2).
@@ -55,7 +55,7 @@ def compute_point_cloud_bbox(point_cloud: np.ndarray, verbose: bool = False) ->
 
 
 def find_all_polygon_bboxes_overlapping_query_bbox(polygon_bboxes: np.ndarray, query_bbox: np.ndarray) -> np.ndarray:
-    """ Find all the overlapping polygon bounding boxes.
+    """Find all the overlapping polygon bounding boxes.
 
     Each bounding box has the following structure:
         bbox = np.array([x_min,y_min,x_max,y_max])
@@ -118,7 +118,7 @@ def find_local_polygons(
     query_min_y: float,
     query_max_y: float,
 ) -> Tuple[np.ndarray, np.ndarray]:
-    """ Find local polygons. We always also return indices.
+    """Find local polygons. We always also return indices.
 
     Take a collection of precomputed polygon bounding boxes, and compare with a query bounding box then returns the
     polygons that overlap, along with their array indices.
@@ -148,7 +148,7 @@ def find_local_polygons(
 def prune_polygons_manhattan_dist(
     query_pt: np.ndarray, points_xyz: np.ndarray, query_search_range_manhattan: float = 200.0
 ) -> np.ndarray:
-    """ Prune polygon points based on a search area defined by the manhattan distance.
+    """Prune polygon points based on a search area defined by the manhattan distance.
 
     Take a collection of small point clouds and return only point clouds that fall within a manhattan search radius of
     the 2D query point.
diff --git a/demo_usage/visualize_30hz_benchmark_data_on_map.py b/demo_usage/visualize_30hz_benchmark_data_on_map.py
index 9100b53b..6390a74c 100644
--- a/demo_usage/visualize_30hz_benchmark_data_on_map.py
+++ b/demo_usage/visualize_30hz_benchmark_data_on_map.py
@@ -45,7 +45,7 @@ def __init__(
     ) -> None:
         """We will cache the accumulated trajectories per city, per log, and per frame
         for the tracking benchmark.
-            """
+        """
         self.plot_lane_tangent_arrows = True
         self.plot_lidar_bev = True
         self.plot_lidar_in_img = False
@@ -311,8 +311,7 @@ def render_front_camera_on_axis(self, ax: plt.Axes, timestamp: int, log_id: str)
 
 
 def visualize_30hz_benchmark_data_on_map(args: Any) -> None:
-    """
-    """
+    """"""
     domv = DatasetOnMapVisualizer(
         args.dataset_dir, args.experiment_prefix, log_id=args.log_id, use_existing_files=args.use_existing_files
     )
diff --git a/integration_tests/test_map_api.py b/integration_tests/test_map_api.py
index 5e8ab2ac..c7dfa240 100644
--- a/integration_tests/test_map_api.py
+++ b/integration_tests/test_map_api.py
@@ -25,16 +25,14 @@ def add_lane_segment_to_ax(
     ymin: float,
     ymax: float,
 ) -> None:
-    """
-        """
+    """"""
     plot_lane_segment_patch(lane_polygon, ax, color=patch_color, alpha=0.3)
 
 
 def find_lane_segment_bounds_in_table(
     avm: ArgoverseMap, city_name: str, lane_segment_id: int
 ) -> Tuple[float, float, float, float]:
-    """
-        """
+    """"""
     match_found = False
     # find the lane segment inside the table
     for table_idx, table_lane_id in avm.city_halluc_tableidx_to_laneid_map[city_name].items():
@@ -49,9 +47,7 @@ def find_lane_segment_bounds_in_table(
 
 
 def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> None:
-    """
-
-        """
+    """"""
     avm = ArgoverseMap()
 
     city_names = ["MIA", "PIT"]
@@ -171,8 +167,7 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> Non
 
 
 def verify_point_in_polygon_for_lanes() -> None:
-    """
-        """
+    """"""
     avm = ArgoverseMap()
 
     # ref_query_x = 422.
@@ -215,8 +210,7 @@ def plot_nearby_halluc_lanes(
     patch_color: str = "r",
     radius: float = 20.0,
 ) -> None:
-    """
-        """
+    """"""
     nearby_lane_ids = avm.get_lane_ids_in_xy_bbox(query_x, query_y, city_name, radius)
     for nearby_lane_id in nearby_lane_ids:
         halluc_lane_polygon = avm.get_lane_segment_polygon(nearby_lane_id, city_name)
@@ -226,11 +220,11 @@ def plot_nearby_halluc_lanes(
 
 def verify_lane_tangent_vector() -> None:
     """
-        debug low confidence lane tangent predictions
+    debug low confidence lane tangent predictions
 
-        I noticed that the confidence score of lane direction is
-        pretty low (almost zero) in some logs
-        """
+    I noticed that the confidence score of lane direction is
+    pretty low (almost zero) in some logs
+    """
     POSE_FILE_DIR = "../debug_lane_tangent"
 
     # both of these are Pittsburgh logs
diff --git a/integration_tests/test_mayavi_utils.py b/integration_tests/test_mayavi_utils.py
index ae19f61d..ff28a007 100644
--- a/integration_tests/test_mayavi_utils.py
+++ b/integration_tests/test_mayavi_utils.py
@@ -29,8 +29,8 @@
 @skip_if_not_mayavi  # type: ignore
 def test_mayavi_import_basic() -> None:
     """
-        To test if Mayavi is installed correctly, generate lines around
-        surface of a torus and render them.
+    To test if Mayavi is installed correctly, generate lines around
+    surface of a torus and render them.
     """
     n_mer, n_long = 6, 11
     pi = np.pi
@@ -194,8 +194,7 @@ def test_draw_lidar_argoverse() -> None:
 
 @skip_if_not_mayavi  # type: ignore
 def test_draw_coordinate_frame_at_origin() -> None:
-    """Test :ref:`draw_coordinate_frame_at_origin`.
-    """
+    """Test :ref:`draw_coordinate_frame_at_origin`."""
     fig = mayavi.mlab.figure(bgcolor=(1, 1, 1), size=(2000, 1000))
     fig = draw_coordinate_frame_at_origin(fig)
     mayavi.mlab.close(fig)
diff --git a/integration_tests/test_plane_visualization_utils.py b/integration_tests/test_plane_visualization_utils.py
index 76f4c11c..59a882c1 100644
--- a/integration_tests/test_plane_visualization_utils.py
+++ b/integration_tests/test_plane_visualization_utils.py
@@ -38,8 +38,7 @@
 
 @skip_if_mayavi_missing  # type: ignore
 def test_plot_frustum_planes_and_normals() -> None:
-    """
-    """
+    """"""
 
     assert planes is not None
     plot_frustum_planes_and_normals(planes, cuboid_verts=None, near_clip_dist=0.5)
@@ -47,8 +46,7 @@ def test_plot_frustum_planes_and_normals() -> None:
 
 @skip_if_mayavi_missing  # type: ignore
 def test_populate_frustum_voxels() -> None:
-    """
-    """
+    """"""
     fig, axis_pair = plt.subplots(1, 1, figsize=(20, 15))
 
     assert planes is not None
@@ -57,7 +55,6 @@ def test_populate_frustum_voxels() -> None:
 
 @skip_if_mayavi_missing  # type: ignore
 def test_get_perpendicular() -> None:
-    """
-    """
+    """"""
     n = [0, 1, 1]
     result = get_perpendicular(n)
diff --git a/integration_tests/test_tracker_eval.py b/integration_tests/test_tracker_eval.py
index 5bb17953..6f31b1ec 100644
--- a/integration_tests/test_tracker_eval.py
+++ b/integration_tests/test_tracker_eval.py
@@ -18,8 +18,8 @@
 
 def test_in_distance_range() -> None:
     """
-        test for in_distance_range_pose()
-        """
+    test for in_distance_range_pose()
+    """
 
     assert eval_tracking.in_distance_range_pose(np.array([0, 0]), np.array([1, 1]), 0, 2)
     assert not eval_tracking.in_distance_range_pose(np.array([0, 0]), np.array([1, 1]), 0, 0.5)
@@ -30,30 +30,30 @@ def test_in_distance_range() -> None:
 
 def test_get_pc_inside_box() -> None:
     """
-        test get_pc_inside_box(pc_raw,bbox) *bbox format is [p0,p1,p2,h]
-
-                    - -------- -
-                   /|         /|
-                  - -------- - . h
-                  | |        | |
-                  . p2 --------
-                  |/         |/
-                 p0 -------- p1
-
-
-        :test lidar data:
-        x    y    z  intensity  laser_number
-        0  0.0  0.0  5.0        4.0          31.0
-        1  1.0  0.0  5.0        1.0          14.0
-        2  2.0  0.0  5.0        0.0          16.0
-        3  3.0  0.0  5.0       20.0          30.0
-        4  4.0  0.0  5.0        3.0          29.0
-        5  5.0  0.0  5.0        1.0          11.0
-        6  6.0  0.0  5.0       31.0          13.0
-        7  7.0  0.0  5.0        2.0          28.0
-        8  8.0  0.0  5.0        5.0          27.0
-        9  9.0  0.0  5.0        6.0          10.0
-        """
+    test get_pc_inside_box(pc_raw,bbox) *bbox format is [p0,p1,p2,h]
+
+                - -------- -
+               /|         /|
+              - -------- - . h
+              | |        | |
+              . p2 --------
+              |/         |/
+             p0 -------- p1
+
+
+    :test lidar data:
+    x    y    z  intensity  laser_number
+    0  0.0  0.0  5.0        4.0          31.0
+    1  1.0  0.0  5.0        1.0          14.0
+    2  2.0  0.0  5.0        0.0          16.0
+    3  3.0  0.0  5.0       20.0          30.0
+    4  4.0  0.0  5.0        3.0          29.0
+    5  5.0  0.0  5.0        1.0          11.0
+    6  6.0  0.0  5.0       31.0          13.0
+    7  7.0  0.0  5.0        2.0          28.0
+    8  8.0  0.0  5.0        5.0          27.0
+    9  9.0  0.0  5.0        6.0          10.0
+    """
     bbox = np.array([np.array([[0], [0], [0]]), np.array([[2], [0], [0]]), np.array([[0], [5], [0]]), np.array(10)])
 
     pc = ply_loader.load_ply(str(TEST_DATA_LOC / "1/lidar/PC_0.ply"))
diff --git a/tests/test_bfs.py b/tests/test_bfs.py
index 00b11c8f..0793126b 100644
--- a/tests/test_bfs.py
+++ b/tests/test_bfs.py
@@ -32,11 +32,11 @@ def compare_paths(paths_lhs: List[List[str]], paths_rhs: List[List[str]]) -> boo
 
 def get_sample_graph() -> Mapping[str, Sequence[str]]:
     """
-        Args:
-            None
+    Args:
+        None
 
-        Returns:
-            graph: Python dictionary representing an adjacency list
+    Returns:
+        graph: Python dictionary representing an adjacency list
     """
     graph = {"1": ["2", "3", "4"], "2": ["5", "6"], "5": ["9", "10"], "4": ["7", "8"], "7": ["11", "12"]}
     return graph
diff --git a/tests/test_centerline_utils.py b/tests/test_centerline_utils.py
index da99bf1c..385d7cf4 100644
--- a/tests/test_centerline_utils.py
+++ b/tests/test_centerline_utils.py
@@ -14,18 +14,18 @@
 
 def temp_test_straight_centerline_to_polygon() -> None:
     """
-        Try converting a simple straight polyline into a polygon. Represents
-        the conversion from a centerline to a lane segment polygon.
-
-        Note that the returned polygon will ba a Numpy array of
-        shape (2N+1,2), with duplicate first and last vertices.
-        Dots below signify the centerline coordinates.
-
-                |   .   |
-                |   .   |
-                |   .   |
-                |   .   |
-        """
+    Try converting a simple straight polyline into a polygon. Represents
+    the conversion from a centerline to a lane segment polygon.
+
+    Note that the returned polygon will ba a Numpy array of
+    shape (2N+1,2), with duplicate first and last vertices.
+    Dots below signify the centerline coordinates.
+
+            |   .   |
+            |   .   |
+            |   .   |
+            |   .   |
+    """
     # create centerline: Numpy array of shape (N,2)
     centerline = np.array([[0, 2.0], [0.0, 0.0], [0.0, -2.0]])
 
diff --git a/tests/test_cuboid_interior.py b/tests/test_cuboid_interior.py
index 076dab3b..4e607059 100644
--- a/tests/test_cuboid_interior.py
+++ b/tests/test_cuboid_interior.py
@@ -81,8 +81,7 @@ def get_scenario_1() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
 
 
 def test_extract_pc_in_box3d_hull() -> None:
-    """
-    """
+    """"""
     pc_raw, bbox_3d, gt_segment, gt_is_valid = get_scenario_1()
     segment, is_valid = extract_pc_in_box3d_hull(pc_raw, bbox_3d)
 
diff --git a/tests/test_cv2_plotting_utils.py b/tests/test_cv2_plotting_utils.py
index b02f2b10..79ea22f7 100644
--- a/tests/test_cv2_plotting_utils.py
+++ b/tests/test_cv2_plotting_utils.py
@@ -12,9 +12,9 @@
 
 def test_draw_point_cloud_in_img_cv2_smokescreen() -> None:
     """
-        We place 4 red circles in an image (with channel order BGR,
-        per the OpenCV convention). We verify that pixel values change accordingly.
-        """
+    We place 4 red circles in an image (with channel order BGR,
+    per the OpenCV convention). We verify that pixel values change accordingly.
+    """
 
     # xy: Numpy array of shape (K,2)
     xy = np.array([[20, 10], [0, 0], [199, 0], [199, 99]])
@@ -36,12 +36,12 @@ def test_draw_point_cloud_in_img_cv2_smokescreen() -> None:
 
 def test_draw_polygon_cv2_smokescreen() -> None:
     """
-        Test ability to fill a nonconvex polygon.
+    Test ability to fill a nonconvex polygon.
 
-        We don't verify the rendered values since this requires
-        scanline rendering computation to find the polygon's
-        exact boundaries on a rasterized grid.
-        """
+    We don't verify the rendered values since this requires
+    scanline rendering computation to find the polygon's
+    exact boundaries on a rasterized grid.
+    """
     UINT8_MAX = 255
     img_w = 40
     img_h = 20
@@ -66,11 +66,11 @@ def test_draw_polygon_cv2_smokescreen() -> None:
 
 def test_plot_bbox_polygon_cv2_smokescreen() -> None:
     """
-        Test drawing a green bounding box, with a thin red border, and
-        test plotted inside, representing a track ID.
+    Test drawing a green bounding box, with a thin red border, and
+    test plotted inside, representing a track ID.
 
-        We don't catch out-of-bounds errors -- that is up to the user.
-        """
+    We don't catch out-of-bounds errors -- that is up to the user.
+    """
     for dtype in [np.uint8, np.float32]:
         bbox_h = 8
         bbox_w = 14
diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py
index c860ba94..e8005d2b 100644
--- a/tests/test_eval_tracking.py
+++ b/tests/test_eval_tracking.py
@@ -46,11 +46,11 @@ def check_mkdir(dirpath: str) -> None:
 
 def yaw_to_quaternion3d(yaw: float) -> Tuple[float, float, float, float]:
     """
-		Args:
-		-   yaw: rotation about the z-axis, in radians
-		Returns:
-		-   qx,qy,qz,qw: quaternion coefficients
-	"""
+    Args:
+    -   yaw: rotation about the z-axis, in radians
+    Returns:
+    -   qx,qy,qz,qw: quaternion coefficients
+    """
     qx, qy, qz, qw = Rotation.from_euler("z", yaw).as_quat()
     return qx, qy, qz, qw
 
@@ -82,9 +82,9 @@ def __init__(self, log_id: str, is_gt: bool) -> None:
 
     def add_obj(self, o: TrackedObjRec, ts_ns: int) -> None:
         """
-			Args:
-			-	ts_ns: timestamp in nanoseconds
-		"""
+        Args:
+            - ts_ns: timestamp in nanoseconds
+        """
         self.ts_to_trackedlabels_dict[ts_ns] += [
             {
                 "center": {"x": o.cx, "y": o.cy, "z": o.cz},
@@ -100,9 +100,9 @@ def add_obj(self, o: TrackedObjRec, ts_ns: int) -> None:
 
     def save_to_disk(self) -> None:
         """
-		Labels and predictions should be saved in JSON e.g.
-			`tracked_object_labels_315969629019741000.json`
-		"""
+        Labels and predictions should be saved in JSON e.g.
+                `tracked_object_labels_315969629019741000.json`
+        """
         for ts_ns, ts_trackedlabels in self.ts_to_trackedlabels_dict.items():
             json_fpath = f"{self.log_dir}/per_sweep_annotations_amodal/"
             check_mkdir(json_fpath)
@@ -114,9 +114,9 @@ def dump_1obj_scenario_json(
     centers: List[Tuple[int, int, int]], yaw_angles: List[float], log_id: str, is_gt: bool
 ) -> None:
     """
-	Egovehicle stationary (represented by `o`).
-	Sequence of 4-nanosecond timestamps.
-	"""
+    Egovehicle stationary (represented by `o`).
+    Sequence of 4-nanosecond timestamps.
+    """
     t_objs = TrackedObjects(log_id=log_id, is_gt=is_gt)
 
     l = 2
@@ -182,25 +182,25 @@ def run_eval(exp_name: str) -> Mapping[str, Any]:
 
 def get_1obj_gt_scenario() -> Tuple[List[Tuple[int, int, int]], List[float]]:
     """
-	Egovehicle stationary (represented by `o`).
-	Seqeuence of 4-nanosecond timestamps.
-
-	|-|
-	| |
-	|-|
-
-	|-|
-	| |
-	|-|
-			o (x,y,z) = (0,0,0)
-	|-|
-	| |
-	|-|
-
-	|-|
-	| | (x,y,z)=(-3,2,0)
-	|-|
-	"""
+    Egovehicle stationary (represented by `o`).
+    Seqeuence of 4-nanosecond timestamps.
+
+    |-|
+    | |
+    |-|
+
+    |-|
+    | |
+    |-|
+                    o (x,y,z) = (0,0,0)
+    |-|
+    | |
+    |-|
+
+    |-|
+    | | (x,y,z)=(-3,2,0)
+    |-|
+    """
     centers = []
     # timestamp 0
     cx = -3
@@ -314,17 +314,17 @@ def test_1obj_offset_translation() -> None:
 
 def test_1obj_poor_translation() -> None:
     """
-	Miss in 1st frame, TP in 2nd frame,
-	lost in 3rd frame, retrack as TP in 4th frame
+    Miss in 1st frame, TP in 2nd frame,
+    lost in 3rd frame, retrack as TP in 4th frame
 
-	Yields 1 fragmentation. Prec=0.5, recall=0.5, F1=0.5
+    Yields 1 fragmentation. Prec=0.5, recall=0.5, F1=0.5
 
-	mostly tracked if it is successfully tracked
-	for at least 80% of its life span
+    mostly tracked if it is successfully tracked
+    for at least 80% of its life span
 
-	If a track is only recovered for less than 20% of its
-	total length, it is said to be mostly lost (ML)
-	"""
+    If a track is only recovered for less than 20% of its
+    total length, it is said to be mostly lost (ML)
+    """
     log_id = "1obj_poor_translation"
 
     centers = []
@@ -512,8 +512,8 @@ def test_orientation_error8() -> None:
 
 def get_mot16_scenario_a() -> Tuple[List[Tuple[int, int, int]], List[float]]:
     """
-	https://arxiv.org/pdf/1603.00831.pdf
-	"""
+    https://arxiv.org/pdf/1603.00831.pdf
+    """
     centers = []
     # timestamp 0
     cx = 0
@@ -557,8 +557,8 @@ def get_mot16_scenario_a() -> Tuple[List[Tuple[int, int, int]], List[float]]:
 
 def test_mot16_scenario_a() -> None:
     """
-	See page 8 of MOT16 paper: https://arxiv.org/pdf/1603.00831.pdf
-	"""
+    See page 8 of MOT16 paper: https://arxiv.org/pdf/1603.00831.pdf
+    """
     log_id = "mot16_scenario_a"
     gt_centers, gt_yaw_angles = get_mot16_scenario_a()
     dump_1obj_scenario_json(gt_centers, gt_yaw_angles, log_id, is_gt=True)
@@ -638,9 +638,9 @@ def test_mot16_scenario_a() -> None:
 
 def test_mot16_scenario_b() -> None:
     """
-	See page 8 of MOT16 paper: https://arxiv.org/pdf/1603.00831.pdf
-	Scenario `a` and Scenario `b` share the same ground truth.
-	"""
+    See page 8 of MOT16 paper: https://arxiv.org/pdf/1603.00831.pdf
+    Scenario `a` and Scenario `b` share the same ground truth.
+    """
     log_id = "mot16_scenario_b"
     gt_centers, gt_yaw_angles = get_mot16_scenario_a()
     dump_1obj_scenario_json(gt_centers, gt_yaw_angles, log_id, is_gt=True)
diff --git a/tests/test_ffmpeg_utils_unit.py b/tests/test_ffmpeg_utils_unit.py
index 65edd2c1..6651233b 100644
--- a/tests/test_ffmpeg_utils_unit.py
+++ b/tests/test_ffmpeg_utils_unit.py
@@ -5,16 +5,14 @@
 
 
 def test_ffmpeg_seq_frame_vid_smokescreen() -> None:
-    """
-        """
+    """"""
     image_prefix = "imgs_%d.jpg"
     output_prefix = "out"
     write_video(image_prefix, output_prefix)
 
 
 def test_ffmpeg_nonseq_frame_vid_smokescreen() -> None:
-    """
-        """
+    """"""
     img_wildcard = "imgs_%*.jpg"
     output_fpath = "out.mp4"
     fps = 10
diff --git a/tests/test_geometry.py b/tests/test_geometry.py
index a6f29d45..2bb13549 100644
--- a/tests/test_geometry.py
+++ b/tests/test_geometry.py
@@ -2,7 +2,11 @@
 
 import numpy as np
 
-from argoverse.utils.geometry import filter_point_cloud_to_polygon, point_inside_polygon, rotate_polygon_about_pt
+from argoverse.utils.geometry import (
+    filter_point_cloud_to_polygon,
+    point_inside_polygon,
+    rotate_polygon_about_pt,
+)
 
 """
 Unit tests for argoverse/utils/geometry.py
@@ -11,8 +15,8 @@
 
 def test_rotate_polygon_about_pt_2d_triangle_0deg_origin() -> None:
     """
-        Rotate a triangle 0 degrees counterclockwise, about the origin
-        """
+    Rotate a triangle 0 degrees counterclockwise, about the origin
+    """
     polygon_pts = np.array([[0, 0], [4, 0], [4, 4]])
     theta = 0  # radians, which is 0 degrees
     c = np.cos(theta)
@@ -25,8 +29,8 @@ def test_rotate_polygon_about_pt_2d_triangle_0deg_origin() -> None:
 
 def test_rotate_polygon_about_pt_2d_triangle_90deg_origin() -> None:
     """
-        Rotate a triangle 90 degrees counterclockwise, about the origin
-        """
+    Rotate a triangle 90 degrees counterclockwise, about the origin
+    """
     polygon_pts = np.array([[0, 0], [4, 0], [4, 4]])
     theta = np.pi / 2  # radians, which is 90 degrees
     c = np.cos(theta)
@@ -41,9 +45,9 @@ def test_rotate_polygon_about_pt_2d_triangle_90deg_origin() -> None:
 
 def test_rotate_polygon_about_pt_2d_triangle_0deg_nonorigin() -> None:
     """
-        Rotate a triangle 0 degrees counterclockwise, but this time
-        not rotating about the origin.
-        """
+    Rotate a triangle 0 degrees counterclockwise, but this time
+    not rotating about the origin.
+    """
     polygon_pts = np.array([[0, 0], [4, 0], [4, 4]])
     theta = 0  # radians, which is 0 degrees
     c = np.cos(theta)
@@ -56,9 +60,9 @@ def test_rotate_polygon_about_pt_2d_triangle_0deg_nonorigin() -> None:
 
 def test_rotate_polygon_about_pt_2d_triangle_90deg_nonorigin() -> None:
     """
-        Rotate a triangle 90 degrees counterclockwise, but this time
-        not rotating about the origin. Instead we rotate about (2,2).
-        """
+    Rotate a triangle 90 degrees counterclockwise, but this time
+    not rotating about the origin. Instead we rotate about (2,2).
+    """
     polygon_pts = np.array([[0, 0], [4, 0], [4, 4]])
     theta = np.pi / 2  # radians, which is 90 degrees
     c = np.cos(theta)
diff --git a/tests/test_interpolate.py b/tests/test_interpolate.py
index e873ef0c..fcdbc3e8 100644
--- a/tests/test_interpolate.py
+++ b/tests/test_interpolate.py
@@ -4,22 +4,27 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from argoverse.utils.interpolate import compute_lane_width, compute_mid_pivot_arc, compute_midpoint_line, interp_arc
+from argoverse.utils.interpolate import (
+    compute_lane_width,
+    compute_mid_pivot_arc,
+    compute_midpoint_line,
+    interp_arc,
+)
 
 
 def test_compute_lane_width_straight() -> None:
     """
-        Compute the lane width of the following straight lane segment
-        (waypoints indicated with "o" symbol):
+    Compute the lane width of the following straight lane segment
+    (waypoints indicated with "o" symbol):
 
-                o   o
-                |   |
-                o   o
-                |   |
-                o   o
+            o   o
+            |   |
+            o   o
+            |   |
+            o   o
 
-        We can swap boundaries for this lane, and the width should be identical.
-        """
+    We can swap boundaries for this lane, and the width should be identical.
+    """
     left_even_pts = np.array([[1, 1], [1, 0], [1, -1]])
     right_even_pts = np.array([[-1, 1], [-1, 0], [-1, -1]])
     lane_width = compute_lane_width(left_even_pts, right_even_pts)
@@ -33,19 +38,19 @@ def test_compute_lane_width_straight() -> None:
 
 def test_compute_lane_width_telescoping() -> None:
     """
-        Compute the lane width of the following straight lane segment
-        (waypoints indicated with "o" symbol):
-
-           o          o
-           \\        //
-                o        o
-                \\     //
-                 o     o
-                  \\ //
-                    o
-
-        We can swap boundaries for this lane, and the width should be identical.
-        """
+    Compute the lane width of the following straight lane segment
+    (waypoints indicated with "o" symbol):
+
+       o          o
+       \\        //
+            o        o
+            \\     //
+             o     o
+              \\ //
+                o
+
+    We can swap boundaries for this lane, and the width should be identical.
+    """
     left_even_pts = np.array([[3, 2], [2, 1], [1, 0], [0, -1]])
     right_even_pts = np.array([[-3, 2], [-2, 1], [-1, 0], [0, -1]])
     lane_width = compute_lane_width(left_even_pts, right_even_pts)
@@ -282,7 +287,7 @@ def test_compute_midpoint_line_curved_maintain_4_waypts() -> None:
     Make sure that if we provide left and right boundary polylines,
     we can get the correct centerline by averaging left and right waypoints.
 
-    Note that because of the curve and the arc interpolation, the land width and centerline in the middle points 
+    Note that because of the curve and the arc interpolation, the land width and centerline in the middle points
     will be shifted.
     """
     right_ln_bnds = np.array([[-1, 3], [1, 3], [4, 0], [4, -2]])
diff --git a/tests/test_mpl_plotting_utils.py b/tests/test_mpl_plotting_utils.py
index 083484d0..ba32727a 100644
--- a/tests/test_mpl_plotting_utils.py
+++ b/tests/test_mpl_plotting_utils.py
@@ -16,8 +16,7 @@
 
 
 def test_draw_polygon_mpl_smokescreen_nolinewidth() -> None:
-    """
-        """
+    """"""
     ax = plt.axes([1, 1, 1, 1])
     # polygon: Numpy array of shape (N,2) or (N,3)
     polygon = np.array([[0, 0], [1, 1], [1, 0], [0, 0]])
@@ -28,8 +27,7 @@ def test_draw_polygon_mpl_smokescreen_nolinewidth() -> None:
 
 
 def test_draw_polygon_mpl_smokescreen_with_linewidth() -> None:
-    """
-        """
+    """"""
     ax = plt.axes([1, 1, 1, 1])
     # polygon: Numpy array of shape (N,2) or (N,3)
     polygon = np.array([[0, 0], [1, 1], [1, 0], [0, 0]])
@@ -41,8 +39,7 @@ def test_draw_polygon_mpl_smokescreen_with_linewidth() -> None:
 
 
 def test_plot_lane_segment_patch_smokescreen() -> None:
-    """
-        """
+    """"""
     ax = plt.axes([1, 1, 1, 1])
     polygon_pts = np.array([[-1, 0], [1, 0], [0, 1]])
     color = "r"
@@ -52,8 +49,7 @@ def test_plot_lane_segment_patch_smokescreen() -> None:
 
 
 def test_plot_nearby_centerlines_smokescreen() -> None:
-    """
-        """
+    """"""
     ax = plt.axes([1, 1, 1, 1])
     # lane_centerlines: Python dictionary where key is lane ID, value is
     # object describing the lane
@@ -73,9 +69,7 @@ def test_plot_nearby_centerlines_smokescreen() -> None:
 
 
 def test_animate_polyline_smokescreen() -> None:
-    """
-        
-        """
+    """"""
     polyline = np.array([[0, 0], [1, 1], [2, 0], [0, 2]])
     axes_margin = 2
     animate_polyline(polyline, axes_margin, show_plot=False)
diff --git a/tests/test_simple_track_dataloader.py b/tests/test_simple_track_dataloader.py
index 45e025c7..f6f94130 100644
--- a/tests/test_simple_track_dataloader.py
+++ b/tests/test_simple_track_dataloader.py
@@ -5,7 +5,9 @@
 
 import pytest
 
-from argoverse.data_loading.simple_track_dataloader import SimpleArgoverseTrackingDataLoader
+from argoverse.data_loading.simple_track_dataloader import (
+    SimpleArgoverseTrackingDataLoader,
+)
 
 _TEST_DATA = pathlib.Path(__file__).parent / "test_data" / "tracking"
 _LOG_ID = "1"
@@ -65,9 +67,9 @@ def test_get_closest_lidar_fpath_found_match(data_loader: SimpleArgoverseTrackin
 
 
 def test_get_closest_lidar_fpath_no_match(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
-    """ LiDAR rotates at 10 Hz (sensor message per 100 ms). Test if camera measurement
-        just barely outside 51 ms allowed buffer. Max LiDAR timestamp in log is 2.
-        51 ms, not 50 ms, is allowed to give time for occasional delay.
+    """LiDAR rotates at 10 Hz (sensor message per 100 ms). Test if camera measurement
+    just barely outside 51 ms allowed buffer. Max LiDAR timestamp in log is 2.
+    51 ms, not 50 ms, is allowed to give time for occasional delay.
     """
     max_allowed_interval = ((100 / 2) + 1) * 1e6
     log_max_lidar_timestamp = 2
diff --git a/tests/test_subprocess_utils.py b/tests/test_subprocess_utils.py
index a2e2309d..478bb350 100644
--- a/tests/test_subprocess_utils.py
+++ b/tests/test_subprocess_utils.py
@@ -7,18 +7,18 @@
 
 def test_run_command_smokescreen() -> None:
     """
-        Do not check output, just verify import works
-        and does not crash.
-        """
+    Do not check output, just verify import works
+    and does not crash.
+    """
     cmd = "echo 5678"
     run_command(cmd)
 
 
 def test_run_command_cat() -> None:
     """
-        Execute a command to dump a string to standard output. Returned
-        output will be in byte format with a carriage return.
-        """
+    Execute a command to dump a string to standard output. Returned
+    output will be in byte format with a carriage return.
+    """
     cmd = "echo 5678"
     stdout_data, stderr_data = run_command(cmd, return_output=True)
     assert stderr_data is None

From bae3b7d726a046b5685e9f844430b6bfa0f6031f Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 19:50:57 -0400
Subject: [PATCH 103/113] Black + isort fix.

---
 .isort.cfg                                    |   1 +
 .../data_loading/argoverse_tracking_loader.py |   6 +-
 .../data_loading/frame_label_accumulator.py   |  24 +++-
 argoverse/data_loading/object_label_record.py |  38 ++++-
 argoverse/data_loading/vector_map_loader.py   |  23 ++-
 argoverse/evaluation/competition_util.py      |  21 ++-
 argoverse/evaluation/detection_utils.py       |  15 +-
 argoverse/evaluation/eval_detection.py        |  13 +-
 argoverse/evaluation/eval_forecasting.py      |  23 ++-
 argoverse/evaluation/eval_tracking.py         |  19 ++-
 argoverse/evaluation/eval_utils.py            |   9 +-
 argoverse/map_representation/map_api.py       | 136 ++++++++++++++----
 .../map_representation/map_viz_helper.py      |   6 +-
 argoverse/utils/calibration.py                |  17 ++-
 argoverse/utils/centerline_utils.py           |   5 +-
 argoverse/utils/city_visibility_utils.py      |  12 +-
 argoverse/utils/cv2_plotting_utils.py         |  22 ++-
 argoverse/utils/forecasting_evaluation.py     |   4 +-
 argoverse/utils/geometry.py                   |   6 +-
 argoverse/utils/grid_interpolation.py         |   5 +-
 argoverse/utils/interpolate.py                |   4 +-
 argoverse/utils/json_utils.py                 |   5 +-
 argoverse/utils/line_projection.py            |   4 +-
 argoverse/utils/make_att_files.py             |  52 +++++--
 argoverse/utils/manhattan_search.py           |   4 +-
 argoverse/utils/mpl_plotting_utils.py         |  24 +++-
 argoverse/utils/plane_visualization_utils.py  |  16 ++-
 argoverse/utils/se2.py                        |   5 +-
 argoverse/utils/se3.py                        |   5 +-
 .../visualization/generate_sequence_videos.py |  25 +++-
 argoverse/visualization/mayavi_utils.py       |  46 +++++-
 .../visualization/mpl_point_cloud_vis.py      |   6 +-
 argoverse/visualization/vis_mask.py           |  14 +-
 .../visualization/visualization_utils.py      |  26 +++-
 .../visualization/visualize_sequences.py      |  25 +++-
 demo_usage/cuboids_to_bboxes.py               |  42 ++++--
 .../visualize_30hz_benchmark_data_on_map.py   |  34 ++++-
 integration_tests/test_map_api.py             |  89 ++++++++++--
 integration_tests/test_mayavi_utils.py        |   6 +-
 .../test_plane_visualization_utils.py         |   3 +-
 integration_tests/test_tracker_eval.py        |   9 +-
 sphinx/conf.py                                |   6 +-
 tests/test_bfs.py                             |  16 ++-
 tests/test_centerline_utils.py                |  35 ++++-
 tests/test_dilate_utils_unit.py               |   9 +-
 tests/test_eval_detection.py                  |  15 +-
 tests/test_eval_tracking.py                   |   5 +-
 tests/test_frustum_clipping.py                |  45 +++++-
 tests/test_geometry.py                        |  38 +++--
 tests/test_grid_interpolation.py              |   9 +-
 tests/test_interpolate.py                     |   7 +-
 tests/test_manhattan_search.py                |  29 +++-
 tests/test_pkl_utils.py                       |  14 +-
 tests/test_se2.py                             |   6 +-
 tests/test_se3.py                             |   7 +-
 tests/test_simple_track_dataloader.py         |  36 +++--
 tests/test_transform.py                       |  36 ++++-
 57 files changed, 957 insertions(+), 205 deletions(-)

diff --git a/.isort.cfg b/.isort.cfg
index 45a8ec06..375c1492 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -2,3 +2,4 @@
 known_first_party=argoverse
 # cat setup.py | grep -Pzo "(?s)install_requires.+?]" | tail -n+2 | head -n-1 | grep -Po '"[a-zA-Z0-9]+' | sed 's/"/,/'
 known_third_party=mayavi,pytest,colour,descartes,imageio,matplotlib,motmetrics,numpy,cv2,pandas,pillow,imageio,pyntcloud,scipy,shapely,sklearn
+line_length = 120
diff --git a/argoverse/data_loading/argoverse_tracking_loader.py b/argoverse/data_loading/argoverse_tracking_loader.py
index 5ea2f6b1..96a04d56 100644
--- a/argoverse/data_loading/argoverse_tracking_loader.py
+++ b/argoverse/data_loading/argoverse_tracking_loader.py
@@ -440,7 +440,11 @@ def get_image_list_sync(self, camera: str, log_id: Optional[str] = None, load: b
         return self._image_list_sync[log_id][camera]
 
     def get_image_at_timestamp(
-        self, timestamp: int, camera: str, log_id: Optional[str] = None, load: bool = True
+        self,
+        timestamp: int,
+        camera: str,
+        log_id: Optional[str] = None,
+        load: bool = True,
     ) -> Optional[Union[str, np.ndarray]]:
         """get image or image path at a specific timestamp
 
diff --git a/argoverse/data_loading/frame_label_accumulator.py b/argoverse/data_loading/frame_label_accumulator.py
index b72bfc98..aebfb1e7 100644
--- a/argoverse/data_loading/frame_label_accumulator.py
+++ b/argoverse/data_loading/frame_label_accumulator.py
@@ -16,10 +16,7 @@
 from argoverse.data_loading.object_label_record import ObjectLabelRecord
 from argoverse.data_loading.pose_loader import get_city_SE3_egovehicle_at_sensor_t
 from argoverse.data_loading.synchronization_database import SynchronizationDB
-from argoverse.data_loading.trajectory_loader import (
-    TrajectoryLabel,
-    load_json_track_labels,
-)
+from argoverse.data_loading.trajectory_loader import TrajectoryLabel, load_json_track_labels
 from argoverse.utils.json_utils import read_json_file
 from argoverse.utils.pkl_utils import load_pkl_dictionary, save_pkl_dictionary
 from argoverse.utils.se3 import SE3
@@ -44,7 +41,12 @@ class PerFrameLabelAccumulator:
     """
 
     def __init__(
-        self, dataset_dir: str, labels_dir: str, experiment_prefix: str, bboxes_3d: bool = False, save: bool = True
+        self,
+        dataset_dir: str,
+        labels_dir: str,
+        experiment_prefix: str,
+        bboxes_3d: bool = False,
+        save: bool = True,
     ) -> None:
         """Initialize PerFrameLabelAccumulator object for use with tracking benchmark data.
 
@@ -165,7 +167,11 @@ def place_trajectory_in_city_frame(self, traj_label: TrajectoryLabel, log_id: st
 
         # store NUM_CUBOID_VERTS (x,y,z) coords per cuboid
         traj_city_fr = np.zeros((seq_len, NUM_CUBOID_VERTS, 3))
-        rand_color = (float(np.random.rand()), float(np.random.rand()), float(np.random.rand()))
+        rand_color = (
+            float(np.random.rand()),
+            float(np.random.rand()),
+            float(np.random.rand()),
+        )
         logger.info(f"On log {log_id} with {traj_label.track_uuid}")
         for t in range(seq_len):
 
@@ -210,7 +216,11 @@ def place_trajectory_in_city_frame(self, traj_label: TrajectoryLabel, log_id: st
         return traj_city_fr
 
     def convert_bbox_to_city_frame(
-        self, lidar_timestamp_ns: int, dataset_dir: str, log_id: str, bbox_ego_frame: np.ndarray
+        self,
+        lidar_timestamp_ns: int,
+        dataset_dir: str,
+        log_id: str,
+        bbox_ego_frame: np.ndarray,
     ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:
         """Convert bounding box to city frame.
         Args:
diff --git a/argoverse/data_loading/object_label_record.py b/argoverse/data_loading/object_label_record.py
index db8d06ba..e09abdd3 100644
--- a/argoverse/data_loading/object_label_record.py
+++ b/argoverse/data_loading/object_label_record.py
@@ -162,14 +162,28 @@ def render_clip_frustum_cv2(
         def draw_rect(selected_corners: np.array, color: Tuple[int, int, int]) -> None:
             prev = selected_corners[-1]
             for corner in selected_corners:
-                draw_clipped_line_segment(img, prev.copy(), corner.copy(), camera_config, linewidth, planes, color)
+                draw_clipped_line_segment(
+                    img,
+                    prev.copy(),
+                    corner.copy(),
+                    camera_config,
+                    linewidth,
+                    planes,
+                    color,
+                )
                 prev = corner
 
         # Draw the sides in green
         for i in range(4):
             # between front and back corners
             draw_clipped_line_segment(
-                img, corners[i], corners[i + 4], camera_config, linewidth, planes, colors[2][::-1]
+                img,
+                corners[i],
+                corners[i + 4],
+                camera_config,
+                linewidth,
+                planes,
+                colors[2][::-1],
             )
 
         # Draw front (first 4 corners) in blue
@@ -181,7 +195,13 @@ def draw_rect(selected_corners: np.array, color: Tuple[int, int, int]) -> None:
         center_bottom_forward = np.mean(corners[2:4], axis=0)
         center_bottom = np.mean(corners[[2, 3, 7, 6]], axis=0)
         draw_clipped_line_segment(
-            img, center_bottom, center_bottom_forward, camera_config, linewidth, planes, colors[0][::-1]
+            img,
+            center_bottom,
+            center_bottom_forward,
+            camera_config,
+            linewidth,
+            planes,
+            colors[0][::-1],
         )
 
         return img
@@ -277,7 +297,17 @@ def json_label_dict_to_obj_record(label: Dict[str, Any]) -> ObjectLabelRecord:
         score = label["score"]
     else:
         score = 1.0
-    obj_rec = ObjectLabelRecord(quaternion, translation, length, width, height, occlusion, label_class, track_id, score)
+    obj_rec = ObjectLabelRecord(
+        quaternion,
+        translation,
+        length,
+        width,
+        height,
+        occlusion,
+        label_class,
+        track_id,
+        score,
+    )
     return obj_rec
 
 
diff --git a/argoverse/data_loading/vector_map_loader.py b/argoverse/data_loading/vector_map_loader.py
index 4a5ab028..1e0ad410 100644
--- a/argoverse/data_loading/vector_map_loader.py
+++ b/argoverse/data_loading/vector_map_loader.py
@@ -44,17 +44,7 @@
 import pathlib
 import sys
 import xml.etree.ElementTree as ET
-from typing import (
-    Any,
-    Dict,
-    List,
-    Mapping,
-    MutableMapping,
-    Optional,
-    Tuple,
-    Union,
-    cast,
-)
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union, cast
 
 import numpy as np
 
@@ -221,7 +211,11 @@ def convert_node_id_list_to_xy(node_id_list: List[int], all_graph_nodes: Mapping
     for i, node_id in enumerate(node_id_list):
         if all_graph_nodes[node_id].height is not None:
             centerline[i] = np.array(
-                [all_graph_nodes[node_id].x, all_graph_nodes[node_id].y, all_graph_nodes[node_id].height]
+                [
+                    all_graph_nodes[node_id].x,
+                    all_graph_nodes[node_id].y,
+                    all_graph_nodes[node_id].height,
+                ]
             )
         else:
             centerline[i] = np.array([all_graph_nodes[node_id].x, all_graph_nodes[node_id].y])
@@ -246,7 +240,10 @@ def extract_node_from_ET_element(child: ET.Element) -> Node:
     node_id = int(node_fields["id"])
     if "height" in node_fields.keys():
         return Node(
-            id=node_id, x=float(node_fields["x"]), y=float(node_fields["y"]), height=float(node_fields["height"])
+            id=node_id,
+            x=float(node_fields["x"]),
+            y=float(node_fields["y"]),
+            height=float(node_fields["height"]),
         )
     return Node(id=node_id, x=float(node_fields["x"]), y=float(node_fields["y"]))
 
diff --git a/argoverse/evaluation/competition_util.py b/argoverse/evaluation/competition_util.py
index c9095289..b1238dfd 100644
--- a/argoverse/evaluation/competition_util.py
+++ b/argoverse/evaluation/competition_util.py
@@ -71,7 +71,12 @@ def generate_forecasting_h5(
 
             d = np.array(
                 [
-                    [key, np.float32(x), np.float32(y), probabilities[key][int(np.floor(i / future_frames))]]
+                    [
+                        key,
+                        np.float32(x),
+                        np.float32(y),
+                        probabilities[key][int(np.floor(i / future_frames))],
+                    ]
                     for i, (x, y) in enumerate(value)
                 ]
             )
@@ -194,7 +199,13 @@ def poly_to_label(poly: Polygon, category: str = "VEHICLE", track_id: str = "")
     # translation = center
     center = np.array([bbox.centroid.xy[0][0], bbox.centroid.xy[1][0], min(z) + height / 2])
 
-    R = np.array([[np.cos(angle), -np.sin(angle), 0], [np.sin(angle), np.cos(angle), 0], [0, 0, 1]])
+    R = np.array(
+        [
+            [np.cos(angle), -np.sin(angle), 0],
+            [np.sin(angle), np.cos(angle), 0],
+            [0, 0, 1],
+        ]
+    )
 
     q = quaternion.from_rotation_matrix(R)
 
@@ -258,7 +269,11 @@ def save_label(argoverse_data: ArgoverseTrackingLoader, labels: List[ObjectLabel
 
     for label in labels:
         json_data = {
-            "center": {"x": label.translation[0], "y": label.translation[1], "z": label.translation[2]},
+            "center": {
+                "x": label.translation[0],
+                "y": label.translation[1],
+                "z": label.translation[2],
+            },
             "rotation": {
                 "x": label.quaternion[0],
                 "y": label.quaternion[1],
diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index e4b43a4b..79313e50 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -132,10 +132,16 @@ def accumulate(
     cls_to_ninst = defaultdict(int)
     for class_name in cfg.dt_classes:
         dt_filtered = filter_instances(
-            dts, class_name, filter_metric=cfg.dt_metric, max_detection_range=cfg.max_dt_range
+            dts,
+            class_name,
+            filter_metric=cfg.dt_metric,
+            max_detection_range=cfg.max_dt_range,
         )
         gt_filtered = filter_instances(
-            gts, class_name, filter_metric=cfg.dt_metric, max_detection_range=cfg.max_dt_range
+            gts,
+            class_name,
+            filter_metric=cfg.dt_metric,
+            max_detection_range=cfg.max_dt_range,
         )
 
         logger.info(f"{dt_filtered.shape[0]} detections")
@@ -217,7 +223,10 @@ def assign(dts: np.ndarray, gts: np.ndarray, cfg: DetectionCfg) -> np.ndarray:
 
 
 def filter_instances(
-    instances: List[ObjectLabelRecord], target_class_name: str, filter_metric: FilterMetric, max_detection_range: float
+    instances: List[ObjectLabelRecord],
+    target_class_name: str,
+    filter_metric: FilterMetric,
+    max_detection_range: float,
 ) -> np.ndarray:
     """Filter the GT annotations based on a set of conditions (class name and distance from egovehicle).
 
diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index b6a564b2..f4f96311 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -181,7 +181,10 @@ def summarize(
                 tp_metrics = self.cfg.tp_normalization_terms
             else:
                 # Calculate TP metrics.
-                tp_metrics = np.mean(cls_stats[:, num_ths : num_ths + N_TP_ERRORS][tp_metrics_mask], axis=0)
+                tp_metrics = np.mean(
+                    cls_stats[:, num_ths : num_ths + N_TP_ERRORS][tp_metrics_mask],
+                    axis=0,
+                )
 
             # Convert errors to scores.
             tp_scores = 1 - (tp_metrics / self.cfg.tp_normalization_terms)
@@ -199,7 +202,13 @@ def summarize(
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("-d", "--dt_fpath", type=str, help="Detection root folder path.", required=True)
-    parser.add_argument("-g", "--gt_fpath", type=str, help="Ground truth root folder path.", required=True)
+    parser.add_argument(
+        "-g",
+        "--gt_fpath",
+        type=str,
+        help="Ground truth root folder path.",
+        required=True,
+    )
     parser.add_argument("-f", "--fig_fpath", type=str, help="Figures root folder path.", default="figs")
     args = parser.parse_args()
     logger.info(f"args == {args}")
diff --git a/argoverse/evaluation/eval_forecasting.py b/argoverse/evaluation/eval_forecasting.py
index f746c5c2..03868edd 100644
--- a/argoverse/evaluation/eval_forecasting.py
+++ b/argoverse/evaluation/eval_forecasting.py
@@ -116,10 +116,18 @@ def get_displacement_errors_and_miss_rate(
         if forecasted_probabilities is not None:
             prob_n_misses.append(1.0 if curr_min_fde > miss_threshold else (1.0 - pruned_probabilities[min_idx]))
             prob_min_ade.append(
-                min(-np.log(pruned_probabilities[min_idx]), -np.log(LOW_PROB_THRESHOLD_FOR_METRICS)) + curr_min_ade
+                min(
+                    -np.log(pruned_probabilities[min_idx]),
+                    -np.log(LOW_PROB_THRESHOLD_FOR_METRICS),
+                )
+                + curr_min_ade
             )
             prob_min_fde.append(
-                min(-np.log(pruned_probabilities[min_idx]), -np.log(LOW_PROB_THRESHOLD_FOR_METRICS)) + curr_min_fde
+                min(
+                    -np.log(pruned_probabilities[min_idx]),
+                    -np.log(LOW_PROB_THRESHOLD_FOR_METRICS),
+                )
+                + curr_min_fde
             )
     metric_results["minADE"] = sum(min_ade) / len(min_ade)
     metric_results["minFDE"] = sum(min_fde) / len(min_fde)
@@ -132,7 +140,9 @@ def get_displacement_errors_and_miss_rate(
 
 
 def get_drivable_area_compliance(
-    forecasted_trajectories: Dict[int, List[np.ndarray]], city_names: Dict[int, str], max_n_guesses: int
+    forecasted_trajectories: Dict[int, List[np.ndarray]],
+    city_names: Dict[int, str],
+    max_n_guesses: int,
 ) -> float:
     """Compute drivable area compliance metric.
 
@@ -190,7 +200,12 @@ def compute_forecasting_metrics(
         metric_results: Dictionary containing values for all metrics.
     """
     metric_results = get_displacement_errors_and_miss_rate(
-        forecasted_trajectories, gt_trajectories, max_n_guesses, horizon, miss_threshold, forecasted_probabilities
+        forecasted_trajectories,
+        gt_trajectories,
+        max_n_guesses,
+        horizon,
+        miss_threshold,
+        forecasted_probabilities,
     )
     metric_results["DAC"] = get_drivable_area_compliance(forecasted_trajectories, city_names, max_n_guesses)
 
diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py
index 10ab960e..187c2fc0 100644
--- a/argoverse/evaluation/eval_tracking.py
+++ b/argoverse/evaluation/eval_tracking.py
@@ -175,7 +175,9 @@ def eval_tracks(
 
             timestamp_lidar = int(Path(path_track_data[ind_frame]).stem.split("_")[-1])
             path_gt = os.path.join(
-                path_dataset, "per_sweep_annotations_amodal", f"tracked_object_labels_{timestamp_lidar}.json"
+                path_dataset,
+                "per_sweep_annotations_amodal",
+                f"tracked_object_labels_{timestamp_lidar}.json",
             )
 
             if not os.path.exists(path_gt):
@@ -198,7 +200,13 @@ def eval_tracks(
 
                 bbox, orientation = label_to_bbox(gt_data[i])
 
-                center = np.array([gt_data[i]["center"]["x"], gt_data[i]["center"]["y"], gt_data[i]["center"]["z"]])
+                center = np.array(
+                    [
+                        gt_data[i]["center"]["x"],
+                        gt_data[i]["center"]["y"],
+                        gt_data[i]["center"]["z"],
+                    ]
+                )
                 if bbox[3] > 0 and in_distance_range_pose(np.zeros(3), center, d_min, d_max):
                     track_label_uuid = gt_data[i]["track_label_uuid"]
                     gt[track_label_uuid] = {}
@@ -333,7 +341,12 @@ def eval_tracks(
         default="../../argodataset_30Hz/test_label/028d5cb1-f74d-366c-85ad-84fde69b0fd3",
     )
     parser.add_argument("--path_dataset", type=str, default="../../argodataset_30Hz/cvpr_test_set")
-    parser.add_argument("--centroid_method", type=str, default="average", choices=["label_center", "average"])
+    parser.add_argument(
+        "--centroid_method",
+        type=str,
+        default="average",
+        choices=["label_center", "average"],
+    )
     parser.add_argument("--flag", type=str, default="")
     parser.add_argument("--d_min", type=float, default=0)
     parser.add_argument("--d_max", type=float, default=100, required=True)
diff --git a/argoverse/evaluation/eval_utils.py b/argoverse/evaluation/eval_utils.py
index 287187cc..8e8ab9df 100644
--- a/argoverse/evaluation/eval_utils.py
+++ b/argoverse/evaluation/eval_utils.py
@@ -100,7 +100,14 @@ def label_to_bbox(label: _LabelType) -> Tuple[np.ndarray, float]:
 
     bbox = np.array([p0, p1, p2, height], dtype=object)
 
-    R = quat2rotmat((label["rotation"]["w"], label["rotation"]["x"], label["rotation"]["y"], label["rotation"]["z"]))
+    R = quat2rotmat(
+        (
+            label["rotation"]["w"],
+            label["rotation"]["x"],
+            label["rotation"]["y"],
+            label["rotation"]["z"],
+        )
+    )
     t = np.array([label["center"]["x"], label["center"]["y"], label["center"]["z"]])[:, np.newaxis]
 
     v = np.array([1, 0, 0])[:, np.newaxis]
diff --git a/argoverse/map_representation/map_api.py b/argoverse/map_representation/map_api.py
index 759c2443..b536d6a2 100644
--- a/argoverse/map_representation/map_api.py
+++ b/argoverse/map_representation/map_api.py
@@ -25,10 +25,7 @@
     find_all_polygon_bboxes_overlapping_query_bbox,
     find_local_polygons,
 )
-from argoverse.utils.mpl_plotting_utils import (
-    plot_lane_segment_patch,
-    visualize_centerline,
-)
+from argoverse.utils.mpl_plotting_utils import plot_lane_segment_patch, visualize_centerline
 from argoverse.utils.pkl_utils import load_pkl_dictionary
 from argoverse.utils.se2 import SE2
 
@@ -64,7 +61,10 @@ def __init__(self) -> None:
         self.im_scale_factor = 50
 
         self.city_lane_centerlines_dict = self.build_centerline_index()
-        self.city_halluc_bbox_table, self.city_halluc_tableidx_to_laneid_map = self.build_hallucinated_lane_bbox_index()
+        (
+            self.city_halluc_bbox_table,
+            self.city_halluc_tableidx_to_laneid_map,
+        ) = self.build_hallucinated_lane_bbox_index()
         self.city_rasterized_da_roi_dict = self.build_city_driveable_area_roi_index()
         self.city_rasterized_ground_height_dict = self.build_city_ground_height_index()
 
@@ -164,7 +164,9 @@ def build_centerline_index(self) -> Mapping[str, Mapping[int, LaneSegment]]:
 
         return city_lane_centerlines_dict
 
-    def build_city_driveable_area_roi_index(self) -> Mapping[str, Mapping[str, np.ndarray]]:
+    def build_city_driveable_area_roi_index(
+        self,
+    ) -> Mapping[str, Mapping[str, np.ndarray]]:
         """
         Load driveable area files from disk. Dilate driveable area to get ROI (takes about 1/2 second).
 
@@ -225,7 +227,10 @@ def get_rasterized_driveable_area(self, city_name: str) -> Tuple[np.ndarray, np.
                     p_city = city_Transformation_pklimage * p_pklimage
         """
         da_mat = self.city_rasterized_da_roi_dict[city_name]["da_mat"]
-        return (da_mat, self.city_rasterized_da_roi_dict[city_name]["npyimage_to_city_se2"])
+        return (
+            da_mat,
+            self.city_rasterized_da_roi_dict[city_name]["npyimage_to_city_se2"],
+        )
 
     def get_rasterized_roi(self, city_name: str) -> Tuple[np.ndarray, np.ndarray]:
         """
@@ -240,9 +245,14 @@ def get_rasterized_roi(self, city_name: str) -> Tuple[np.ndarray, np.ndarray]:
                     p_city = city_Transformation_pklimage * p_pklimage
         """
         roi_mat = self.city_rasterized_da_roi_dict[city_name]["roi_mat"]
-        return (roi_mat, self.city_rasterized_da_roi_dict[city_name]["npyimage_to_city_se2"])
+        return (
+            roi_mat,
+            self.city_rasterized_da_roi_dict[city_name]["npyimage_to_city_se2"],
+        )
 
-    def build_hallucinated_lane_bbox_index(self) -> Tuple[Mapping[str, Any], Mapping[str, Any]]:
+    def build_hallucinated_lane_bbox_index(
+        self,
+    ) -> Tuple[Mapping[str, Any], Mapping[str, Any]]:
         """
         Populate the pre-computed hallucinated extent of each lane polygon, to allow for fast
         queries.
@@ -288,7 +298,10 @@ def get_rasterized_ground_height(self, city_name: str) -> Tuple[np.ndarray, np.n
                     p_city = city_Transformation_pklimage * p_pklimage
         """
         ground_height_mat = self.city_rasterized_ground_height_dict[city_name]["ground_height"]
-        return (ground_height_mat, self.city_rasterized_ground_height_dict[city_name]["npyimage_to_city_se2"])
+        return (
+            ground_height_mat,
+            self.city_rasterized_ground_height_dict[city_name]["npyimage_to_city_se2"],
+        )
 
     def remove_ground_surface(
         self, point_cloud: np.ndarray, city_name: str, return_logicals: bool = False
@@ -424,7 +437,10 @@ def get_raster_layer_points_boolean(self, point_cloud: np.ndarray, city_name: st
         if layer_name == "roi":
             layer_raster_mat, npyimage_to_city_se2_mat = self.get_rasterized_roi(city_name)
         elif layer_name == "driveable_area":
-            layer_raster_mat, npyimage_to_city_se2_mat = self.get_rasterized_driveable_area(city_name)
+            (
+                layer_raster_mat,
+                npyimage_to_city_se2_mat,
+            ) = self.get_rasterized_driveable_area(city_name)
         else:
             raise ValueError("layer_name should be wither roi or driveable_area.")
 
@@ -567,10 +583,24 @@ def get_lane_direction(
         lane_dir_vector = next_waypoint - prev_waypoint
         if visualize:
             plt.plot(centerline[:, 0], centerline[:, 1], color="y")
-            plt.scatter(query_xy_city_coords[0], query_xy_city_coords[1], 200, marker=".", color="b")
+            plt.scatter(
+                query_xy_city_coords[0],
+                query_xy_city_coords[1],
+                200,
+                marker=".",
+                color="b",
+            )
             dx = lane_dir_vector[0] * 10
             dy = lane_dir_vector[1] * 10
-            plt.arrow(query_xy_city_coords[0], query_xy_city_coords[1], dx, dy, color="r", width=0.3, zorder=2)
+            plt.arrow(
+                query_xy_city_coords[0],
+                query_xy_city_coords[1],
+                dx,
+                dy,
+                color="r",
+                width=0.3,
+                zorder=2,
+            )
             centerline_length = centerline.shape[0]
             for i in range(centerline_length):
                 plt.scatter(centerline[i, 0], centerline[i, 1], i / 5.0, marker=".", color="k")
@@ -581,7 +611,11 @@ def get_lane_direction(
         return lane_dir_vector, confidence
 
     def get_lane_ids_in_xy_bbox(
-        self, query_x: float, query_y: float, city_name: str, query_search_range_manhattan: float = 5.0
+        self,
+        query_x: float,
+        query_y: float,
+        city_name: str,
+        query_search_range_manhattan: float = 5.0,
     ) -> List[int]:
         """
         Prune away all lane segments based on Manhattan distance. We vectorize this instead
@@ -610,7 +644,8 @@ def get_lane_ids_in_xy_bbox(
         query_max_y = query_y + query_search_range_manhattan
 
         overlap_indxs = find_all_polygon_bboxes_overlapping_query_bbox(
-            self.city_halluc_bbox_table[city_name], np.array([query_min_x, query_min_y, query_max_x, query_max_y])
+            self.city_halluc_bbox_table[city_name],
+            np.array([query_min_x, query_min_y, query_max_x, query_max_y]),
         )
 
         if len(overlap_indxs) == 0:
@@ -791,7 +826,11 @@ def get_cl_from_lane_seq(self, lane_seqs: Iterable[List[int]], city_name: str) -
         return candidate_cl
 
     def get_candidate_centerlines_for_traj(
-        self, xy: np.ndarray, city_name: str, viz: bool = False, max_search_radius: float = 50.0
+        self,
+        xy: np.ndarray,
+        city_name: str,
+        viz: bool = False,
+        max_search_radius: float = 50.0,
     ) -> List[np.ndarray]:
         """Get centerline candidates upto a threshold. .
 
@@ -856,7 +895,15 @@ def get_candidate_centerlines_for_traj(
             plt.figure(0, figsize=(8, 7))
             for centerline_coords in candidate_centerlines:
                 visualize_centerline(centerline_coords)
-            plt.plot(xy[:, 0], xy[:, 1], "-", color="#d33e4c", alpha=1, linewidth=1, zorder=15)
+            plt.plot(
+                xy[:, 0],
+                xy[:, 1],
+                "-",
+                color="#d33e4c",
+                alpha=1,
+                linewidth=1,
+                zorder=15,
+            )
 
             final_x = xy[-1, 0]
             final_y = xy[-1, 1]
@@ -906,7 +953,13 @@ def dfs(
                 for child in child_lanes:
                     centerline = self.get_lane_segment_centerline(child, city_name)
                     cl_length = LineString(centerline).length
-                    curr_lane_ids = self.dfs(child, city_name, dist + cl_length, threshold, extend_along_predecessor)
+                    curr_lane_ids = self.dfs(
+                        child,
+                        city_name,
+                        dist + cl_length,
+                        threshold,
+                        extend_along_predecessor,
+                    )
                     traversed_lanes.extend(curr_lane_ids)
             if len(traversed_lanes) == 0:
                 return [[lane_id]]
@@ -926,9 +979,18 @@ def draw_lane(self, lane_segment_id: int, city_name: str, legend: bool = False)
         """
         lane_segment_polygon = self.get_lane_segment_polygon(lane_segment_id, city_name)
         if legend:
-            plt.plot(lane_segment_polygon[:, 0], lane_segment_polygon[:, 1], color="dimgray", label=lane_segment_id)
+            plt.plot(
+                lane_segment_polygon[:, 0],
+                lane_segment_polygon[:, 1],
+                color="dimgray",
+                label=lane_segment_id,
+            )
         else:
-            plt.plot(lane_segment_polygon[:, 0], lane_segment_polygon[:, 1], color="lightgrey")
+            plt.plot(
+                lane_segment_polygon[:, 0],
+                lane_segment_polygon[:, 1],
+                color="lightgrey",
+            )
         plt.axis("equal")
 
     def get_lane_segments_containing_xy(self, query_x: float, query_y: float, city_name: str) -> List[int]:
@@ -954,14 +1016,24 @@ def get_lane_segments_containing_xy(self, query_x: float, query_y: float, city_n
             for lane_id in neighborhood_lane_ids:
                 lane_polygon = self.get_lane_segment_polygon(lane_id, city_name)
                 inside = point_inside_polygon(
-                    lane_polygon.shape[0], lane_polygon[:, 0], lane_polygon[:, 1], query_x, query_y
+                    lane_polygon.shape[0],
+                    lane_polygon[:, 0],
+                    lane_polygon[:, 1],
+                    query_x,
+                    query_y,
                 )
                 if inside:
                     occupied_lane_ids += [lane_id]
         return occupied_lane_ids
 
     def plot_nearby_halluc_lanes(
-        self, ax: plt.Axes, city_name: str, query_x: float, query_y: float, patch_color: str = "r", radius: float = 20
+        self,
+        ax: plt.Axes,
+        city_name: str,
+        query_x: float,
+        query_y: float,
+        patch_color: str = "r",
+        radius: float = 20,
     ) -> None:
         """
         Plot lane segment for nearby lanes of the specified x, y location
@@ -992,7 +1064,12 @@ def find_local_lane_polygons(self, query_bbox: Tuple[float, float, float, float]
         lane_bboxes = self.city_to_lane_bboxes_dict[city_name]
         xmin, xmax, ymin, ymax = query_bbox
         local_lane_polygons, _ = find_local_polygons(
-            copy.deepcopy(lane_polygons), copy.deepcopy(lane_bboxes), xmin, xmax, ymin, ymax
+            copy.deepcopy(lane_polygons),
+            copy.deepcopy(lane_bboxes),
+            xmin,
+            xmax,
+            ymin,
+            ymax,
         )
         return local_lane_polygons
 
@@ -1012,12 +1089,21 @@ def find_local_driveable_areas(self, query_bbox: Tuple[float, float, float, floa
         da_bboxes = self.city_to_da_bboxes_dict[city_name]
         xmin, xmax, ymin, ymax = query_bbox
         local_das, _ = find_local_polygons(
-            copy.deepcopy(driveable_areas), copy.deepcopy(da_bboxes), xmin, xmax, ymin, ymax
+            copy.deepcopy(driveable_areas),
+            copy.deepcopy(da_bboxes),
+            xmin,
+            xmax,
+            ymin,
+            ymax,
         )
         return local_das
 
     def find_local_lane_centerlines(
-        self, query_x: float, query_y: float, city_name: str, query_search_range_manhattan: float = 80.0
+        self,
+        query_x: float,
+        query_y: float,
+        city_name: str,
+        query_search_range_manhattan: float = 80.0,
     ) -> np.ndarray:
         """
         Find local lane centerline to the specified x,y location
diff --git a/argoverse/map_representation/map_viz_helper.py b/argoverse/map_representation/map_viz_helper.py
index be709806..f568afcc 100644
--- a/argoverse/map_representation/map_viz_helper.py
+++ b/argoverse/map_representation/map_viz_helper.py
@@ -196,7 +196,11 @@ def render_global_city_map_bev(
             color = blue
 
         draw_polyline_cv2(
-            centerline_2d * UPSAMPLE_FACTOR, rendered_image, color, im_h * UPSAMPLE_FACTOR, im_w * UPSAMPLE_FACTOR
+            centerline_2d * UPSAMPLE_FACTOR,
+            rendered_image,
+            color,
+            im_h * UPSAMPLE_FACTOR,
+            im_w * UPSAMPLE_FACTOR,
         )
 
     # provide colormap in corner
diff --git a/argoverse/utils/calibration.py b/argoverse/utils/calibration.py
index 93d018bb..d49d3589 100644
--- a/argoverse/utils/calibration.py
+++ b/argoverse/utils/calibration.py
@@ -231,7 +231,10 @@ def load_calib(calib_filepath: Union[str, Path]) -> Dict[Any, Calibration]:
     calib_list = {}
     for camera in CAMERA_LIST:
         cam_config = get_calibration_config(calib, camera)
-        calib_cam = next((c for c in calib["camera_data_"] if c["key"] == f"image_raw_{camera}"), None)
+        calib_cam = next(
+            (c for c in calib["camera_data_"] if c["key"] == f"image_raw_{camera}"),
+            None,
+        )
 
         if calib_cam is None:
             continue
@@ -520,7 +523,10 @@ def distort_single(radius_undist: float, distort_coeffs: List[float]) -> float:
 
 
 def project_lidar_to_undistorted_img(
-    lidar_points_h: np.ndarray, calib_data: Dict[str, Any], camera_name: str, remove_nan: bool = False
+    lidar_points_h: np.ndarray,
+    calib_data: Dict[str, Any],
+    camera_name: str,
+    remove_nan: bool = False,
 ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, CameraConfig]:
     camera_config = get_calibration_config(calib_data, camera_name)
 
@@ -543,7 +549,12 @@ def project_lidar_to_undistorted_img(
 # uv_cam: Numpy array of shape (N,3) with dtype np.float32
 # valid_pts_bool: Numpy array of shape (N,) with dtype bool
 # camera configuration : (only if you asked for it).
-_ReturnWithOptConfig = Tuple[Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray], Optional[CameraConfig]]
+_ReturnWithOptConfig = Tuple[
+    Optional[np.ndarray],
+    Optional[np.ndarray],
+    Optional[np.ndarray],
+    Optional[CameraConfig],
+]
 _ReturnWithoutOptConfig = Tuple[Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray]]
 
 
diff --git a/argoverse/utils/centerline_utils.py b/argoverse/utils/centerline_utils.py
index 1c6fd770..e788915a 100644
--- a/argoverse/utils/centerline_utils.py
+++ b/argoverse/utils/centerline_utils.py
@@ -123,7 +123,10 @@ def convert_lane_boundaries_to_polygon(right_lane_bounds: np.ndarray, left_lane_
 
 
 def filter_candidate_centerlines(
-    xy: np.ndarray, candidate_cl: List[np.ndarray], stationary_threshold: float = 2.0, max_dist_margin: float = 2.0
+    xy: np.ndarray,
+    candidate_cl: List[np.ndarray],
+    stationary_threshold: float = 2.0,
+    max_dist_margin: float = 2.0,
 ) -> List[np.ndarray]:
     """Filter candidate centerlines based on the distance travelled along the centerline.
 
diff --git a/argoverse/utils/city_visibility_utils.py b/argoverse/utils/city_visibility_utils.py
index 34393d6d..5b6975c9 100644
--- a/argoverse/utils/city_visibility_utils.py
+++ b/argoverse/utils/city_visibility_utils.py
@@ -55,7 +55,9 @@ def clip_point_cloud_to_visible_region(
         visualize = False
         if visualize:
             viz_polar_bin_contents(
-                bin_lidar_pts, egovehicle_pts[invalid_egovehicle_bools], filename=f"polar_bin_{i}.jpg"
+                bin_lidar_pts,
+                egovehicle_pts[invalid_egovehicle_bools],
+                filename=f"polar_bin_{i}.jpg",
             )
 
         egovehicle_pts = egovehicle_pts[np.logical_not(invalid_egovehicle_bools)]
@@ -73,7 +75,13 @@ def viz_polar_bin_contents(bin_lidar_pts: np.ndarray, invalid_egovehicle_pts: np
     """
     # visualize the contents of this polar bin
     plt.scatter(bin_lidar_pts[:, 0], bin_lidar_pts[:, 1], 10, marker=".", color="b")
-    plt.scatter(invalid_egovehicle_pts[:, 0], invalid_egovehicle_pts[:, 1], 10, marker=".", color="r")
+    plt.scatter(
+        invalid_egovehicle_pts[:, 0],
+        invalid_egovehicle_pts[:, 1],
+        10,
+        marker=".",
+        color="r",
+    )
     plt.axis("equal")
     plt.savefig(filename)
     plt.close("all")
diff --git a/argoverse/utils/cv2_plotting_utils.py b/argoverse/utils/cv2_plotting_utils.py
index 5bad3a68..44af0685 100644
--- a/argoverse/utils/cv2_plotting_utils.py
+++ b/argoverse/utils/cv2_plotting_utils.py
@@ -39,7 +39,13 @@ def draw_clipped_line_segment(
 
     uv_a = uv_a.squeeze()
     uv_b = uv_b.squeeze()
-    cv2.line(img, (int(uv_a[0]), int(uv_a[1])), (int(uv_b[0]), int(uv_b[1])), color, linewidth)
+    cv2.line(
+        img,
+        (int(uv_a[0]), int(uv_a[1])),
+        (int(uv_b[0]), int(uv_b[1])),
+        color,
+        linewidth,
+    )
 
 
 def draw_point_cloud_in_img_cv2(img: np.ndarray, xy: np.ndarray, colors: np.ndarray, radius: int = 5) -> np.ndarray:
@@ -64,7 +70,11 @@ def draw_point_cloud_in_img_cv2(img: np.ndarray, xy: np.ndarray, colors: np.ndar
 
 
 def draw_polyline_cv2(
-    line_segments_arr: np.ndarray, image: np.ndarray, color: Tuple[int, int, int], im_h: int, im_w: int
+    line_segments_arr: np.ndarray,
+    image: np.ndarray,
+    color: Tuple[int, int, int],
+    im_h: int,
+    im_w: int,
 ) -> None:
     """Draw a polyline onto an image using given line segments.
 
@@ -137,7 +147,13 @@ def plot_bbox_polygon_cv2(img: np.ndarray, track_id: str, color: np.ndarray, bbo
     plot_x = xmin + 10
     plot_y = ymin + 25
     img = cv2.putText(
-        img, str(track_id), (plot_x, plot_y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, thickness=5, color=white
+        img,
+        str(track_id),
+        (plot_x, plot_y),
+        fontFace=cv2.FONT_HERSHEY_SIMPLEX,
+        fontScale=1,
+        thickness=5,
+        color=white,
     )
 
     red = (255, 0, 0)
diff --git a/argoverse/utils/forecasting_evaluation.py b/argoverse/utils/forecasting_evaluation.py
index d9995456..9e40bf83 100644
--- a/argoverse/utils/forecasting_evaluation.py
+++ b/argoverse/utils/forecasting_evaluation.py
@@ -16,7 +16,9 @@ def compute_summed_distance_point_cloud2D(points_a: np.ndarray, points_b: np.nda
 
 
 def evaluate_prediction(
-    pred_traj: np.ndarray, ground_truth_traj: np.ndarray, eval_method: str = "EVAL_DESTINATION_ONLY"
+    pred_traj: np.ndarray,
+    ground_truth_traj: np.ndarray,
+    eval_method: str = "EVAL_DESTINATION_ONLY",
 ) -> np.ndarray:
     """Compute the error as L2 norm in trajectories
 
diff --git a/argoverse/utils/geometry.py b/argoverse/utils/geometry.py
index 6a812caf..c71a0a60 100644
--- a/argoverse/utils/geometry.py
+++ b/argoverse/utils/geometry.py
@@ -52,7 +52,11 @@ def filter_point_cloud_to_polygon(polygon: np.ndarray, point_cloud: np.ndarray)
 
 
 def point_inside_polygon(
-    n_vertices: int, poly_x_pts: np.ndarray, poly_y_pts: np.ndarray, test_x: float, test_y: float
+    n_vertices: int,
+    poly_x_pts: np.ndarray,
+    poly_y_pts: np.ndarray,
+    test_x: float,
+    test_y: float,
 ) -> bool:
     """Check whether a point is inside a polygon.
 
diff --git a/argoverse/utils/grid_interpolation.py b/argoverse/utils/grid_interpolation.py
index 9d65a0ed..0539b5ba 100644
--- a/argoverse/utils/grid_interpolation.py
+++ b/argoverse/utils/grid_interpolation.py
@@ -5,7 +5,10 @@
 
 
 def interp_square_grid(
-    grid_data: np.ndarray, in_dim: int = 200, out_dim: int = 30, interp_type: str = "linear"
+    grid_data: np.ndarray,
+    in_dim: int = 200,
+    out_dim: int = 30,
+    interp_type: str = "linear",
 ) -> np.ndarray:
     """
     Interpolate a square grid
diff --git a/argoverse/utils/interpolate.py b/argoverse/utils/interpolate.py
index 72b22985..edc5e348 100644
--- a/argoverse/utils/interpolate.py
+++ b/argoverse/utils/interpolate.py
@@ -48,7 +48,9 @@ def compute_mid_pivot_arc(single_pt: np.ndarray, arc_pts: np.ndarray) -> np.ndar
 
 
 def compute_midpoint_line(
-    left_ln_bnds: np.ndarray, right_ln_bnds: np.ndarray, num_interp_pts: int = NUM_CENTERLINE_INTERP_PTS
+    left_ln_bnds: np.ndarray,
+    right_ln_bnds: np.ndarray,
+    num_interp_pts: int = NUM_CENTERLINE_INTERP_PTS,
 ) -> np.ndarray:
     """
     Compute the lane segment centerline by interpolating n points along each
diff --git a/argoverse/utils/json_utils.py b/argoverse/utils/json_utils.py
index b25eced8..d2d50f54 100644
--- a/argoverse/utils/json_utils.py
+++ b/argoverse/utils/json_utils.py
@@ -19,7 +19,10 @@ def read_json_file(fpath: Union[str, "os.PathLike[str]"]) -> Any:
         return json.load(f)
 
 
-def save_json_dict(json_fpath: Union[str, "os.PathLike[str]"], dictionary: Union[Dict[Any, Any], List[Any]]) -> None:
+def save_json_dict(
+    json_fpath: Union[str, "os.PathLike[str]"],
+    dictionary: Union[Dict[Any, Any], List[Any]],
+) -> None:
     """Save a Python dictionary to a JSON file.
 
     Args:
diff --git a/argoverse/utils/line_projection.py b/argoverse/utils/line_projection.py
index 06552ce9..5168e084 100644
--- a/argoverse/utils/line_projection.py
+++ b/argoverse/utils/line_projection.py
@@ -33,7 +33,9 @@ def project_to_line_seq(
 
 
 def project_to_line(
-    trajectory: np.ndarray, center_polyline: np.ndarray, enforce_same_density: bool = False
+    trajectory: np.ndarray,
+    center_polyline: np.ndarray,
+    enforce_same_density: bool = False,
 ) -> Tuple[float, np.ndarray]:
     """Project a trajectory onto a polyline.
 
diff --git a/argoverse/utils/make_att_files.py b/argoverse/utils/make_att_files.py
index e8295c28..4bab7ed0 100644
--- a/argoverse/utils/make_att_files.py
+++ b/argoverse/utils/make_att_files.py
@@ -69,7 +69,12 @@ def save_bev_img(
 
     for bbox, difficulty_att in zip(list_bboxes, list_difficulty_att):
 
-        qw, qx, qy, qz = bbox["rotation"]["w"], bbox["rotation"]["x"], bbox["rotation"]["y"], bbox["rotation"]["z"]
+        qw, qx, qy, qz = (
+            bbox["rotation"]["w"],
+            bbox["rotation"]["x"],
+            bbox["rotation"]["y"],
+            bbox["rotation"]["z"],
+        )
         theta_local = np.arctan2(2 * (qw * qz + qx * qy), 1 - 2 * (qy * qy + qz * qz))
         pose_local = np.array([bbox["center"]["x"], bbox["center"]["y"], bbox["center"]["z"]])
 
@@ -83,9 +88,20 @@ def save_bev_img(
         color = (color_0, color_1, color_2)
 
         w, l, h = bbox["width"], bbox["length"], bbox["height"]
-        bbox_2d = np.array([[-l / 2, -w / 2, 0], [l / 2, -w / 2, 0], [-l / 2, w / 2, 0], [l / 2, w / 2, 0]])
+        bbox_2d = np.array(
+            [
+                [-l / 2, -w / 2, 0],
+                [l / 2, -w / 2, 0],
+                [-l / 2, w / 2, 0],
+                [l / 2, w / 2, 0],
+            ]
+        )
         R = np.array(
-            [[np.cos(theta_local), -np.sin(theta_local), 0], [np.sin(theta_local), np.cos(theta_local), 0], [0, 0, 1]]
+            [
+                [np.cos(theta_local), -np.sin(theta_local), 0],
+                [np.sin(theta_local), np.cos(theta_local), 0],
+                [0, 0, 1],
+            ]
         )
         bbox_2d = np.matmul(R, bbox_2d.transpose()).transpose() + pose_local[0:3]
         edge_2d = np.array([[0, 1], [0, 2], [2, 3], [1, 3]])
@@ -115,11 +131,22 @@ def save_bev_img(
 
     offset = 0
     for key in dict_color.keys():
-        cv2.putText(img, key, (100 + offset, image_size - 50), font, fontScale, dict_color[key], lineType)
+        cv2.putText(
+            img,
+            key,
+            (100 + offset, image_size - 50),
+            font,
+            fontScale,
+            dict_color[key],
+            lineType,
+        )
         offset += 150
 
     print("Saving img: ", path_imgs)
-    cv2.imwrite(os.path.join(path_imgs, "%s_%s_%d.jpg" % (dataset_name, log_id, lidar_timestamp)), img * 255)
+    cv2.imwrite(
+        os.path.join(path_imgs, "%s_%s_%d.jpg" % (dataset_name, log_id, lidar_timestamp)),
+        img * 255,
+    )
 
 
 def bspline_1d(x: np.array, y: np.array, s: float = 20.0, k: int = 3) -> np.array:
@@ -153,7 +180,13 @@ def derivative(x: np.array) -> np.array:
         dx/dt: N-length Numpy array, with derivative of x w.r.t. timestep
     """
     x_tensor = torch.Tensor(x).unsqueeze(0).unsqueeze(0)
-    x_padded = torch.cat((x_tensor, (x_tensor[:, :, -1] - x_tensor[:, :, -2] + x_tensor[:, :, -1]).unsqueeze(0)), dim=2)
+    x_padded = torch.cat(
+        (
+            x_tensor,
+            (x_tensor[:, :, -1] - x_tensor[:, :, -2] + x_tensor[:, :, -1]).unsqueeze(0),
+        ),
+        dim=2,
+    )
     filters = torch.Tensor([-1, 1]).unsqueeze(0).unsqueeze(0)
 
     return F.conv1d(x_padded, filters)[0, 0].numpy()
@@ -305,9 +338,10 @@ def make_att_files(root_dir: str) -> None:
                 ), "zero-length track"
                 dict_tracks[id_track]["length_track"] = length_track
 
-                dict_tracks[id_track]["list_vel"], dict_tracks[id_track]["list_acc"] = compute_v_a(
-                    dict_tracks[id_track]["list_center_w"]
-                )
+                (
+                    dict_tracks[id_track]["list_vel"],
+                    dict_tracks[id_track]["list_acc"],
+                ) = compute_v_a(dict_tracks[id_track]["list_center_w"])
                 dict_tracks[id_track]["num_missing"] = (
                     dict_tracks[id_track]["length_track"] - dict_tracks[id_track]["exists"].sum()
                 )
diff --git a/argoverse/utils/manhattan_search.py b/argoverse/utils/manhattan_search.py
index b363451d..88277b8e 100644
--- a/argoverse/utils/manhattan_search.py
+++ b/argoverse/utils/manhattan_search.py
@@ -146,7 +146,9 @@ def find_local_polygons(
 
 
 def prune_polygons_manhattan_dist(
-    query_pt: np.ndarray, points_xyz: np.ndarray, query_search_range_manhattan: float = 200.0
+    query_pt: np.ndarray,
+    points_xyz: np.ndarray,
+    query_search_range_manhattan: float = 200.0,
 ) -> np.ndarray:
     """Prune polygon points based on a search area defined by the manhattan distance.
 
diff --git a/argoverse/utils/mpl_plotting_utils.py b/argoverse/utils/mpl_plotting_utils.py
index 77e95231..61f622b8 100644
--- a/argoverse/utils/mpl_plotting_utils.py
+++ b/argoverse/utils/mpl_plotting_utils.py
@@ -12,7 +12,10 @@
 
 
 def draw_polygon_mpl(
-    ax: plt.Axes, polygon: np.ndarray, color: Union[Tuple[float, float, float], str], linewidth: Optional[float] = None
+    ax: plt.Axes,
+    polygon: np.ndarray,
+    color: Union[Tuple[float, float, float], str],
+    linewidth: Optional[float] = None,
 ) -> None:
     """Draw a polygon.
 
@@ -47,7 +50,9 @@ def draw_polygonpatch_matplotlib(points: Any, color: Union[Tuple[float, float, f
 
 
 def draw_lane_polygons(
-    ax: plt.Axes, lane_polygons: np.ndarray, color: Union[Tuple[float, float, float], str] = "y"
+    ax: plt.Axes,
+    lane_polygons: np.ndarray,
+    color: Union[Tuple[float, float, float], str] = "y",
 ) -> None:
     """Draw a lane using polygons.
 
@@ -61,7 +66,10 @@ def draw_lane_polygons(
 
 
 def plot_bbox_2D(
-    ax: plt.Axes, pts: np.ndarray, color: Union[Tuple[float, float, float], str], linestyle: str = "-"
+    ax: plt.Axes,
+    pts: np.ndarray,
+    color: Union[Tuple[float, float, float], str],
+    linestyle: str = "-",
 ) -> None:
     """Draw a bounding box.
 
@@ -120,7 +128,10 @@ def update(frame: List[Any]) -> Tuple[Line2D]:
 
 
 def plot_lane_segment_patch(
-    polygon_pts: np.ndarray, ax: plt.Axes, color: Union[Tuple[float, float, float], str] = "y", alpha: float = 0.3
+    polygon_pts: np.ndarray,
+    ax: plt.Axes,
+    color: Union[Tuple[float, float, float], str] = "y",
+    alpha: float = 0.3,
 ) -> None:
     """Plot a lane segment using a PolygonPatch.
 
@@ -136,7 +147,10 @@ def plot_lane_segment_patch(
 
 
 def plot_nearby_centerlines(
-    lane_centerlines: Dict[Any, Any], ax: plt.Axes, nearby_lane_ids: List[int], color: Union[Tuple[int, int, int], str]
+    lane_centerlines: Dict[Any, Any],
+    ax: plt.Axes,
+    nearby_lane_ids: List[int],
+    color: Union[Tuple[int, int, int], str],
 ) -> None:
     """Plot centerlines.
 
diff --git a/argoverse/utils/plane_visualization_utils.py b/argoverse/utils/plane_visualization_utils.py
index 978e8dcf..1d4de6ab 100644
--- a/argoverse/utils/plane_visualization_utils.py
+++ b/argoverse/utils/plane_visualization_utils.py
@@ -52,7 +52,9 @@ def populate_frustum_voxels(planes: List[np.ndarray], fig: Figure, axis_pair: st
 
 
 def plot_frustum_planes_and_normals(
-    planes: List[np.ndarray], cuboid_verts: Optional[np.ndarray] = None, near_clip_dist: float = 0.5
+    planes: List[np.ndarray],
+    cuboid_verts: Optional[np.ndarray] = None,
+    near_clip_dist: float = 0.5,
 ) -> None:
     """
     Args:
@@ -89,7 +91,17 @@ def plot_frustum_planes_and_normals(
         plane_pts = generate_grid_on_plane(a, b, c, d, P)
         fig = plot_points_3D_mayavi(plane_pts, fig, color)
         # plot the normals at (0,0,0.5) and normal vector (u,v,w) given by (a,b,c)
-        mayavi_wrapper.mlab.quiver3d(0, 0, 0.5, a * 1000, b * 1000, c * 1000, color=color, figure=fig, line_width=8)
+        mayavi_wrapper.mlab.quiver3d(
+            0,
+            0,
+            0.5,
+            a * 1000,
+            b * 1000,
+            c * 1000,
+            color=color,
+            figure=fig,
+            line_width=8,
+        )
 
     # draw teal line at top below the camera
     pt1 = np.array([-5, 0, -5])
diff --git a/argoverse/utils/se2.py b/argoverse/utils/se2.py
index 50037226..61704537 100644
--- a/argoverse/utils/se2.py
+++ b/argoverse/utils/se2.py
@@ -75,5 +75,8 @@ def right_multiply_with_se2(self, right_se2: "SE2") -> "SE2":
             The composed transformation.
         """
         chained_transform_matrix = self.transform_matrix.dot(right_se2.transform_matrix)
-        chained_se2 = SE2(rotation=chained_transform_matrix[:2, :2], translation=chained_transform_matrix[:2, 2])
+        chained_se2 = SE2(
+            rotation=chained_transform_matrix[:2, :2],
+            translation=chained_transform_matrix[:2, 2],
+        )
         return chained_se2
diff --git a/argoverse/utils/se3.py b/argoverse/utils/se3.py
index 0a7aae5c..6366e710 100644
--- a/argoverse/utils/se3.py
+++ b/argoverse/utils/se3.py
@@ -67,5 +67,8 @@ def right_multiply_with_se3(self, right_se3: "SE3") -> "SE3":
             chained_se3: instance of SE3 class
         """
         chained_transform_matrix = self.transform_matrix.dot(right_se3.transform_matrix)
-        chained_se3 = SE3(rotation=chained_transform_matrix[:3, :3], translation=chained_transform_matrix[:3, 3])
+        chained_se3 = SE3(
+            rotation=chained_transform_matrix[:3, :3],
+            translation=chained_transform_matrix[:3, 3],
+        )
         return chained_se3
diff --git a/argoverse/visualization/generate_sequence_videos.py b/argoverse/visualization/generate_sequence_videos.py
index 200f6083..41436e21 100644
--- a/argoverse/visualization/generate_sequence_videos.py
+++ b/argoverse/visualization/generate_sequence_videos.py
@@ -23,7 +23,10 @@
 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 parser.add_argument("--root", help="sequence location ")
 parser.add_argument(
-    "--max_videos", type=int, help="maximum number of sequence to process. -1 to get all sequence", default=10
+    "--max_videos",
+    type=int,
+    help="maximum number of sequence to process. -1 to get all sequence",
+    default=10,
 )
 parser.add_argument("--output_dir", help="output directory", default="vis_video")
 
@@ -78,13 +81,27 @@ def main(arguments: List[str]) -> int:
             df_cur = df.loc[df["TIMESTAMP"] <= time]
 
             if time == time_list[-1]:
-                viz_sequence(df_cur, lane_centerlines=lane_centerlines, show=False, smoothen=False)
+                viz_sequence(
+                    df_cur,
+                    lane_centerlines=lane_centerlines,
+                    show=False,
+                    smoothen=False,
+                )
             else:
-                viz_sequence(df_cur, lane_centerlines=lane_centerlines, show=False, smoothen=False)
+                viz_sequence(
+                    df_cur,
+                    lane_centerlines=lane_centerlines,
+                    show=False,
+                    smoothen=False,
+                )
 
             os.makedirs(seq_out_dir, exist_ok=True)
 
-            plt.savefig(os.path.join(seq_out_dir, f"{count}.png"), bbox_inches="tight", pad_inches=0)
+            plt.savefig(
+                os.path.join(seq_out_dir, f"{count}.png"),
+                bbox_inches="tight",
+                pad_inches=0,
+            )
             plt.close()
             count += 1
 
diff --git a/argoverse/visualization/mayavi_utils.py b/argoverse/visualization/mayavi_utils.py
index 7a72929a..611ebd26 100644
--- a/argoverse/visualization/mayavi_utils.py
+++ b/argoverse/visualization/mayavi_utils.py
@@ -87,7 +87,13 @@ def draw_rect(fig: Figure, selected_corners: np.ndarray, color: Color) -> None:
 
     if draw_text:
         mayavi_wrapper.mlab.text3d(
-            corners[0, 0], corners[0, 1], corners[0, 2], draw_text, scale=text_scale, color=colors[0], figure=fig
+            corners[0, 0],
+            corners[0, 1],
+            corners[0, 2],
+            draw_text,
+            scale=text_scale,
+            color=colors[0],
+            figure=fig,
         )
 
     # Draw the sides in green
@@ -219,21 +225,39 @@ def draw_coordinate_frame_at_origin(fig: Figure) -> Figure:
     axes = np.array([[2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 2.0]], dtype=np.float64)
     # e_1 in red
     mayavi_wrapper.mlab.plot3d(
-        [0, axes[0, 0]], [0, axes[0, 1]], [0, axes[0, 2]], color=(1, 0, 0), tube_radius=None, figure=fig
+        [0, axes[0, 0]],
+        [0, axes[0, 1]],
+        [0, axes[0, 2]],
+        color=(1, 0, 0),
+        tube_radius=None,
+        figure=fig,
     )
     # e_2 in green
     mayavi_wrapper.mlab.plot3d(
-        [0, axes[1, 0]], [0, axes[1, 1]], [0, axes[1, 2]], color=(0, 1, 0), tube_radius=None, figure=fig
+        [0, axes[1, 0]],
+        [0, axes[1, 1]],
+        [0, axes[1, 2]],
+        color=(0, 1, 0),
+        tube_radius=None,
+        figure=fig,
     )
     # e_3 in blue
     mayavi_wrapper.mlab.plot3d(
-        [0, axes[2, 0]], [0, axes[2, 1]], [0, axes[2, 2]], color=(0, 0, 1), tube_radius=None, figure=fig
+        [0, axes[2, 0]],
+        [0, axes[2, 1]],
+        [0, axes[2, 2]],
+        color=(0, 0, 1),
+        tube_radius=None,
+        figure=fig,
     )
     return fig
 
 
 def draw_lidar(
-    point_cloud: np.ndarray, colormap: str = "spectral", fig: Optional[Figure] = None, bgcolor: Color = (0, 0, 0)
+    point_cloud: np.ndarray,
+    colormap: str = "spectral",
+    fig: Optional[Figure] = None,
+    bgcolor: Color = (0, 0, 0),
 ) -> Figure:
     """Render a :ref:`PointCloud` with a 45 degree viewing frustum from ego-vehicle.
 
@@ -263,11 +287,19 @@ def draw_lidar(
 
     # draw points
     fig = plot_points_3D_mayavi(
-        points=point_cloud, fig=fig, per_pt_color_strengths=thresholded_heights, fixed_color=None, colormap=colormap
+        points=point_cloud,
+        fig=fig,
+        per_pt_color_strengths=thresholded_heights,
+        fixed_color=None,
+        colormap=colormap,
     )
     fig = draw_coordinate_frame_at_origin(fig)
     mayavi_wrapper.mlab.view(
-        azimuth=180, elevation=70, focalpoint=[12.0909996, -1.04700089, -2.03249991], distance=62.0, figure=fig
+        azimuth=180,
+        elevation=70,
+        focalpoint=[12.0909996, -1.04700089, -2.03249991],
+        distance=62.0,
+        figure=fig,
     )
     return fig
 
diff --git a/argoverse/visualization/mpl_point_cloud_vis.py b/argoverse/visualization/mpl_point_cloud_vis.py
index 3f179b63..729df7f4 100644
--- a/argoverse/visualization/mpl_point_cloud_vis.py
+++ b/argoverse/visualization/mpl_point_cloud_vis.py
@@ -9,7 +9,11 @@
 __all__ = ["draw_point_cloud_bev"]
 
 AXIS_INDEX = [0, 1]
-AXIS_LIMITS = [[-80, 80], [-90, 90], [-10, 50]]  # X axis range  # Y axis range  # Z axis range
+AXIS_LIMITS = [
+    [-80, 80],
+    [-90, 90],
+    [-10, 50],
+]  # X axis range  # Y axis range  # Z axis range
 AXIS_NAME = ["X", "Y", "Z"]
 
 
diff --git a/argoverse/visualization/vis_mask.py b/argoverse/visualization/vis_mask.py
index 4c66f120..19443cc8 100644
--- a/argoverse/visualization/vis_mask.py
+++ b/argoverse/visualization/vis_mask.py
@@ -58,7 +58,12 @@ def vis_mask(image: np.ndarray, mask: np.ndarray, color: float, alpha: float = 0
     return image.astype(np.uint8)
 
 
-def vis_class(image: np.ndarray, pos: Tuple[float, float], class_str: str, font_scale: float = 50.0) -> np.ndarray:
+def vis_class(
+    image: np.ndarray,
+    pos: Tuple[float, float],
+    class_str: str,
+    font_scale: float = 50.0,
+) -> np.ndarray:
     """Visualizes a class.
 
     Args:
@@ -270,7 +275,12 @@ def vis_one_image(
 
             for contour in contours:
                 polygon = Polygon(
-                    contour.reshape((-1, 2)), fill=True, facecolor=color_mask, edgecolor="w", linewidth=1.2, alpha=0.5
+                    contour.reshape((-1, 2)),
+                    fill=True,
+                    facecolor=color_mask,
+                    edgecolor="w",
+                    linewidth=1.2,
+                    alpha=0.5,
                 )
                 ax.add_patch(polygon)
 
diff --git a/argoverse/visualization/visualization_utils.py b/argoverse/visualization/visualization_utils.py
index 81948146..598bbea5 100644
--- a/argoverse/visualization/visualization_utils.py
+++ b/argoverse/visualization/visualization_utils.py
@@ -16,7 +16,11 @@
 from argoverse.utils.frustum_clipping import generate_frustum_planes
 
 point_size = 0.01
-axes_limits = [[-10, 10], [-10, 10], [-3, 10]]  # X axis range  # Y axis range  # Z axis range
+axes_limits = [
+    [-10, 10],
+    [-10, 10],
+    [-3, 10],
+]  # X axis range  # Y axis range  # Z axis range
 axes_str = ["X", "Y", "Z"]
 
 _COLOR_MAP = [
@@ -98,7 +102,11 @@ def draw_point_cloud_trajectory(
         for label in argoverse_data.get_label_object(i):
             unique_id_list.add(label.track_id)
     color_map = {
-        track_id: (float(np.random.rand()), float(np.random.rand()), float(np.random.rand()))
+        track_id: (
+            float(np.random.rand()),
+            float(np.random.rand()),
+            float(np.random.rand()),
+        )
         for track_id in unique_id_list
     }
     pc = argoverse_data.get_lidar(idx)
@@ -162,7 +170,13 @@ def draw_point_cloud_trajectory(
 
     for track_id in traj_by_id.keys():
         traj = np.array(traj_by_id[track_id])
-        ax.plot(traj[:, 0], traj[:, 1], color=color_map[track_id], linestyle="--", linewidth=1)
+        ax.plot(
+            traj[:, 0],
+            traj[:, 1],
+            color=color_map[track_id],
+            linestyle="--",
+            linewidth=1,
+        )
 
 
 def draw_box(
@@ -209,7 +223,11 @@ def show_image_with_boxes(img: np.ndarray, objects: Iterable[ObjectLabelRecord],
         uv_cam = calib.project_ego_to_cam(box3d_pts_3d)
 
         img1 = obj.render_clip_frustum_cv2(
-            img1, uv_cam[:, :3], planes.copy(), copy.deepcopy(calib.camera_config), linewidth=3
+            img1,
+            uv_cam[:, :3],
+            planes.copy(),
+            copy.deepcopy(calib.camera_config),
+            linewidth=3,
         )
 
     return img1
diff --git a/argoverse/visualization/visualize_sequences.py b/argoverse/visualization/visualize_sequences.py
index 851b1d4b..7ab208d1 100644
--- a/argoverse/visualization/visualize_sequences.py
+++ b/argoverse/visualization/visualize_sequences.py
@@ -35,7 +35,10 @@ def interpolate_polyline(polyline: np.ndarray, num_points: int) -> np.ndarray:
 
 
 def viz_sequence(
-    df: pd.DataFrame, lane_centerlines: Optional[np.ndarray] = None, show: bool = True, smoothen: bool = False
+    df: pd.DataFrame,
+    lane_centerlines: Optional[np.ndarray] = None,
+    show: bool = True,
+    smoothen: bool = False,
 ) -> None:
 
     # Seq data
@@ -75,7 +78,15 @@ def viz_sequence(
                 lane_centerlines.append(lane_cl)
 
     for lane_cl in lane_centerlines:
-        plt.plot(lane_cl[:, 0], lane_cl[:, 1], "--", color="grey", alpha=1, linewidth=1, zorder=0)
+        plt.plot(
+            lane_cl[:, 0],
+            lane_cl[:, 1],
+            "--",
+            color="grey",
+            alpha=1,
+            linewidth=1,
+            zorder=0,
+        )
     frames = df.groupby("TRACK_ID")
 
     plt.xlabel("Map X")
@@ -136,7 +147,15 @@ def viz_sequence(
         object_type_tracker[object_type] += 1
 
     red_star = mlines.Line2D([], [], color="red", marker="*", linestyle="None", markersize=7, label="Agent")
-    green_circle = mlines.Line2D([], [], color="green", marker="o", linestyle="None", markersize=7, label="Others")
+    green_circle = mlines.Line2D(
+        [],
+        [],
+        color="green",
+        marker="o",
+        linestyle="None",
+        markersize=7,
+        label="Others",
+    )
     black_triangle = mlines.Line2D([], [], color="black", marker="^", linestyle="None", markersize=7, label="AV")
 
     plt.axis("off")
diff --git a/demo_usage/cuboids_to_bboxes.py b/demo_usage/cuboids_to_bboxes.py
index 5275842a..b329733b 100644
--- a/demo_usage/cuboids_to_bboxes.py
+++ b/demo_usage/cuboids_to_bboxes.py
@@ -88,7 +88,13 @@ def plot_lane_centerlines_in_img(
         centerline_uv_cam = clip_point_cloud_to_visible_region(centerline_uv_cam, lidar_pts)
         for i in range(centerline_uv_cam.shape[0] - 1):
             draw_clipped_line_segment(
-                img, centerline_uv_cam[i], centerline_uv_cam[i + 1], camera_config, linewidth, planes, color
+                img,
+                centerline_uv_cam[i],
+                centerline_uv_cam[i + 1],
+                camera_config,
+                linewidth,
+                planes,
+                color,
             )
     return img
 
@@ -165,7 +171,13 @@ def dump_clipped_3d_cuboids_to_images(
                 camera_config = get_calibration_config(log_calib_data, camera_name)
                 planes = generate_frustum_planes(camera_config.intrinsic.copy(), camera_name)
                 img = plot_lane_centerlines_in_img(
-                    lidar_pts, city_to_egovehicle_se3, img, city_name, avm, camera_config, planes
+                    lidar_pts,
+                    city_to_egovehicle_se3,
+                    img,
+                    city_name,
+                    avm,
+                    camera_config,
+                    planes,
                 )
 
                 for label_idx, label in enumerate(labels):
@@ -176,7 +188,7 @@ def dump_clipped_3d_cuboids_to_images(
                     cuboid_vertices = obj_rec.as_3d_bbox()
                     points_h = point_cloud_to_homogeneous(cuboid_vertices).T
                     if motion_compensate:
-                        uv, uv_cam, valid_pts_bool, K = project_lidar_to_img_motion_compensated(
+                        (uv, uv_cam, valid_pts_bool, K,) = project_lidar_to_img_motion_compensated(
                             points_h,  # these are recorded at lidar_time
                             copy.deepcopy(log_calib_data),
                             camera_name,
@@ -188,14 +200,20 @@ def dump_clipped_3d_cuboids_to_images(
                         )
                     else:
                         # project_lidar_to_img
-                        uv, uv_cam, valid_pts_bool, camera_config = project_lidar_to_undistorted_img(
-                            points_h, copy.deepcopy(log_calib_data), camera_name
-                        )
+                        (
+                            uv,
+                            uv_cam,
+                            valid_pts_bool,
+                            camera_config,
+                        ) = project_lidar_to_undistorted_img(points_h, copy.deepcopy(log_calib_data), camera_name)
                     if valid_pts_bool.sum() == 0:
                         continue
 
                     img = obj_rec.render_clip_frustum_cv2(
-                        img, uv_cam.T[:, :3], planes.copy(), copy.deepcopy(camera_config)
+                        img,
+                        uv_cam.T[:, :3],
+                        planes.copy(),
+                        copy.deepcopy(camera_config),
                     )
 
                 cv2.imwrite(save_img_fpath, img)
@@ -227,7 +245,10 @@ def main(args: Any):
     """Run the example."""
     log_ids = [log_id.strip() for log_id in args.log_ids.split(",")]
     dump_clipped_3d_cuboids_to_images(
-        log_ids, args.max_num_images_to_render * 9, args.dataset_dir, args.experiment_prefix
+        log_ids,
+        args.max_num_images_to_render * 9,
+        args.dataset_dir,
+        args.experiment_prefix,
     )
 
 
@@ -235,7 +256,10 @@ def main(args: Any):
     # Parse command line arguments
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--max-num-images-to-render", default=5, type=int, help="number of images within which to render 3d cuboids"
+        "--max-num-images-to-render",
+        default=5,
+        type=int,
+        help="number of images within which to render 3d cuboids",
     )
     parser.add_argument("--dataset-dir", type=str, required=True, help="path to the dataset folder")
     parser.add_argument(
diff --git a/demo_usage/visualize_30hz_benchmark_data_on_map.py b/demo_usage/visualize_30hz_benchmark_data_on_map.py
index 6390a74c..866c7f6f 100644
--- a/demo_usage/visualize_30hz_benchmark_data_on_map.py
+++ b/demo_usage/visualize_30hz_benchmark_data_on_map.py
@@ -41,7 +41,11 @@
 
 class DatasetOnMapVisualizer:
     def __init__(
-        self, dataset_dir: str, experiment_prefix: str, use_existing_files: bool = True, log_id: str = None
+        self,
+        dataset_dir: str,
+        experiment_prefix: str,
+        use_existing_files: bool = True,
+        log_id: str = None,
     ) -> None:
         """We will cache the accumulated trajectories per city, per log, and per frame
         for the tracking benchmark.
@@ -135,7 +139,10 @@ def plot_log_one_at_a_time(self, log_id="", idx=-1, save_video=True, city=""):
                     ycenter = pose_city_to_ego["translation"][1]
                     ego_center_xyz = np.array(pose_city_to_ego["translation"])
 
-                    city_to_egovehicle_se3 = SE3(rotation=pose_city_to_ego["rotation"], translation=ego_center_xyz)
+                    city_to_egovehicle_se3 = SE3(
+                        rotation=pose_city_to_ego["rotation"],
+                        translation=ego_center_xyz,
+                    )
 
                     if self.plot_lidar_bev:
                         xmin = xcenter - 80  # 150
@@ -281,11 +288,20 @@ def render_bev_labels_mpl(
                         if self.plot_lane_tangent_arrows:
                             bbox_center = np.mean(bbox_city_fr, axis=0)
                             tangent_xy, conf = avm.get_lane_direction(
-                                query_xy_city_coords=bbox_center[:2], city_name=city_name
+                                query_xy_city_coords=bbox_center[:2],
+                                city_name=city_name,
                             )
                             dx = tangent_xy[0] * LANE_TANGENT_VECTOR_SCALING
                             dy = tangent_xy[1] * LANE_TANGENT_VECTOR_SCALING
-                            ax.arrow(bbox_center[0], bbox_center[1], dx, dy, color="r", width=0.5, zorder=2)
+                            ax.arrow(
+                                bbox_center[0],
+                                bbox_center[1],
+                                dx,
+                                dy,
+                                color="r",
+                                width=0.5,
+                                zorder=2,
+                            )
                     else:
                         plot_bbox_2D(ax, bbox_ego_frame, color)
                         cuboid_lidar_pts, _ = filter_point_cloud_to_bbox_2D_vectorized(
@@ -313,7 +329,10 @@ def render_front_camera_on_axis(self, ax: plt.Axes, timestamp: int, log_id: str)
 def visualize_30hz_benchmark_data_on_map(args: Any) -> None:
     """"""
     domv = DatasetOnMapVisualizer(
-        args.dataset_dir, args.experiment_prefix, log_id=args.log_id, use_existing_files=args.use_existing_files
+        args.dataset_dir,
+        args.experiment_prefix,
+        log_id=args.log_id,
+        use_existing_files=args.use_existing_files,
     )
     # Plotting does not work on AWS! The figure cannot be refreshed properly
     # Thus, plotting must be performed locally.
@@ -331,7 +350,10 @@ def visualize_30hz_benchmark_data_on_map(args: Any) -> None:
         help="results will be saved in a folder with this prefix for its name",
     )
     parser.add_argument(
-        "--log_id", default=None, type=str, help="log ids, this is the folder name in argoverse-tracking/*/[log_id]"
+        "--log_id",
+        default=None,
+        type=str,
+        help="log ids, this is the folder name in argoverse-tracking/*/[log_id]",
     )
     parser.add_argument(
         "--use_existing_files",
diff --git a/integration_tests/test_map_api.py b/integration_tests/test_map_api.py
index c7dfa240..c3935a9f 100644
--- a/integration_tests/test_map_api.py
+++ b/integration_tests/test_map_api.py
@@ -76,7 +76,16 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> Non
                     lane_centerline = avm.get_lane_segment_centerline(predecessor_id, city_name)
                     halluc_lane_polygon = avm.get_lane_segment_polygon(predecessor_id, city_name)
                     xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, predecessor_id)
-                    add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "r", xmin, xmax, ymin, ymax)
+                    add_lane_segment_to_ax(
+                        ax,
+                        lane_centerline,
+                        halluc_lane_polygon,
+                        "r",
+                        xmin,
+                        xmax,
+                        ymin,
+                        ymax,
+                    )
 
             if successor_ids is not None:
                 # add successors
@@ -84,21 +93,48 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> Non
                     lane_centerline = avm.get_lane_segment_centerline(successor_id, city_name)
                     halluc_lane_polygon = avm.get_lane_segment_polygon(successor_id, city_name)
                     xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, successor_id)
-                    add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "b", xmin, xmax, ymin, ymax)
+                    add_lane_segment_to_ax(
+                        ax,
+                        lane_centerline,
+                        halluc_lane_polygon,
+                        "b",
+                        xmin,
+                        xmax,
+                        ymin,
+                        ymax,
+                    )
 
             # add left neighbor
             if l_neighbor_id is not None:
                 lane_centerline = avm.get_lane_segment_centerline(l_neighbor_id, city_name)
                 halluc_lane_polygon = avm.get_lane_segment_polygon(l_neighbor_id, city_name)
                 xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, l_neighbor_id)
-                add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "g", xmin, xmax, ymin, ymax)
+                add_lane_segment_to_ax(
+                    ax,
+                    lane_centerline,
+                    halluc_lane_polygon,
+                    "g",
+                    xmin,
+                    xmax,
+                    ymin,
+                    ymax,
+                )
 
             # add right neighbor
             if r_neighbor_id is not None:
                 lane_centerline = avm.get_lane_segment_centerline(r_neighbor_id, city_name)
                 halluc_lane_polygon = avm.get_lane_segment_polygon(r_neighbor_id, city_name)
                 xmin, ymin, xmax, ymax = find_lane_segment_bounds_in_table(avm, city_name, r_neighbor_id)
-                add_lane_segment_to_ax(ax, lane_centerline, halluc_lane_polygon, "m", xmin, xmax, ymin, ymax)
+                add_lane_segment_to_ax(
+                    ax,
+                    lane_centerline,
+                    halluc_lane_polygon,
+                    "m",
+                    xmin,
+                    xmax,
+                    ymin,
+                    ymax,
+                )
 
             if enable_lane_boundaries:
                 # Compare with Argo's proprietary, ground truth lane boundaries
@@ -106,7 +142,13 @@ def verify_halluc_lane_extent_index(enable_lane_boundaries: bool = False) -> Non
                 for gt_lane_polygon in gt_lane_polygons:
                     dist = np.linalg.norm(gt_lane_polygon.mean(axis=0)[:2] - np.array([xmin, ymin]))
                     if dist < 30:
-                        ax.plot(gt_lane_polygon[:, 0], gt_lane_polygon[:, 1], color="k", alpha=0.3, zorder=1)
+                        ax.plot(
+                            gt_lane_polygon[:, 0],
+                            gt_lane_polygon[:, 1],
+                            color="k",
+                            alpha=0.3,
+                            zorder=1,
+                        )
 
             ax.axis("equal")
             plt.show()
@@ -215,7 +257,11 @@ def plot_nearby_halluc_lanes(
     for nearby_lane_id in nearby_lane_ids:
         halluc_lane_polygon = avm.get_lane_segment_polygon(nearby_lane_id, city_name)
         plot_lane_segment_patch(halluc_lane_polygon, ax, color=patch_color, alpha=0.3)
-        plt.text(halluc_lane_polygon[:, 0].mean(), halluc_lane_polygon[:, 1].mean(), str(nearby_lane_id))
+        plt.text(
+            halluc_lane_polygon[:, 0].mean(),
+            halluc_lane_polygon[:, 1].mean(),
+            str(nearby_lane_id),
+        )
 
 
 def verify_lane_tangent_vector() -> None:
@@ -228,7 +274,10 @@ def verify_lane_tangent_vector() -> None:
     POSE_FILE_DIR = "../debug_lane_tangent"
 
     # both of these are Pittsburgh logs
-    log_ids = ["033669d3-3d6b-3d3d-bd93-7985d86653ea", "028d5cb1-f74d-366c-85ad-84fde69b0fd3"]
+    log_ids = [
+        "033669d3-3d6b-3d3d-bd93-7985d86653ea",
+        "028d5cb1-f74d-366c-85ad-84fde69b0fd3",
+    ]
 
     avm = ArgoverseMap()
     city_name = "PIT"
@@ -267,7 +316,15 @@ def verify_lane_tangent_vector() -> None:
 
                 dx = lane_dir_vector[0] * 20
                 dy = lane_dir_vector[1] * 20
-                plt.arrow(query_xy_city_coords[0], query_xy_city_coords[1], dx, dy, color="r", width=0.3, zorder=2)
+                plt.arrow(
+                    query_xy_city_coords[0],
+                    query_xy_city_coords[1],
+                    dx,
+                    dy,
+                    color="r",
+                    width=0.3,
+                    zorder=2,
+                )
 
                 query_x, query_y = query_xy_city_coords
                 ax.scatter([query_x], [query_y], 100, color="k", marker=".")
@@ -283,7 +340,11 @@ def verify_lane_tangent_vector() -> None:
 def test_remove_extended_predecessors() -> None:
     """Test remove_extended_predecessors() for map_api"""
 
-    lane_seqs = [[9621385, 9619110, 9619209, 9631133], [9621385, 9619110, 9619209], [9619209, 9631133]]
+    lane_seqs = [
+        [9621385, 9619110, 9619209, 9631133],
+        [9621385, 9619110, 9619209],
+        [9619209, 9631133],
+    ]
     xy = np.array([[-130.0, 2315.0], [-129.0, 2315.0], [-128.0, 2315.0]])  # 9619209 comntains xy[0]
     city_name = "MIA"
 
@@ -313,7 +374,15 @@ def test_get_candidate_centerlines_for_traj() -> None:
                         (CL3)                               2310
     """
     xy = np.array(
-        [[-130.0, 2315.0], [-129.0, 2315.0], [-128.0, 2315.0], [-127, 2315], [-126, 2315], [-125, 2315], [-124, 2315]]
+        [
+            [-130.0, 2315.0],
+            [-129.0, 2315.0],
+            [-128.0, 2315.0],
+            [-127, 2315],
+            [-126, 2315],
+            [-125, 2315],
+            [-124, 2315],
+        ]
     )
     city_name = "MIA"
     avm = ArgoverseMap()
diff --git a/integration_tests/test_mayavi_utils.py b/integration_tests/test_mayavi_utils.py
index ff28a007..b700ce54 100644
--- a/integration_tests/test_mayavi_utils.py
+++ b/integration_tests/test_mayavi_utils.py
@@ -149,7 +149,11 @@ def test_plot_bbox_3d_mayavi_drawtext() -> None:
 
     for cuboid_verts in [cuboid_1_verts, cuboid_2_verts]:
         fig = plot_bbox_3d_mayavi(
-            fig, cuboid_verts, line_width=linewidth, draw_text="box 0th vertex is here", text_scale=(0.1, 0.1, 0.1)
+            fig,
+            cuboid_verts,
+            line_width=linewidth,
+            draw_text="box 0th vertex is here",
+            text_scale=(0.1, 0.1, 0.1),
         )
     mayavi.mlab.close(fig)
 
diff --git a/integration_tests/test_plane_visualization_utils.py b/integration_tests/test_plane_visualization_utils.py
index 59a882c1..e2b0ffdb 100644
--- a/integration_tests/test_plane_visualization_utils.py
+++ b/integration_tests/test_plane_visualization_utils.py
@@ -32,7 +32,8 @@
 
 
 skip_if_mayavi_missing = pytest.mark.skipif(
-    MAYAVI_MISSING, reason="Could not test functionality that depends on mayavi because mayavi is missing."
+    MAYAVI_MISSING,
+    reason="Could not test functionality that depends on mayavi because mayavi is missing.",
 )
 
 
diff --git a/integration_tests/test_tracker_eval.py b/integration_tests/test_tracker_eval.py
index 6f31b1ec..a45450ae 100644
--- a/integration_tests/test_tracker_eval.py
+++ b/integration_tests/test_tracker_eval.py
@@ -54,7 +54,14 @@ def test_get_pc_inside_box() -> None:
     8  8.0  0.0  5.0        5.0          27.0
     9  9.0  0.0  5.0        6.0          10.0
     """
-    bbox = np.array([np.array([[0], [0], [0]]), np.array([[2], [0], [0]]), np.array([[0], [5], [0]]), np.array(10)])
+    bbox = np.array(
+        [
+            np.array([[0], [0], [0]]),
+            np.array([[2], [0], [0]]),
+            np.array([[0], [5], [0]]),
+            np.array(10),
+        ]
+    )
 
     pc = ply_loader.load_ply(str(TEST_DATA_LOC / "1/lidar/PC_0.ply"))
 
diff --git a/sphinx/conf.py b/sphinx/conf.py
index 9ffea2e4..6c00bac3 100644
--- a/sphinx/conf.py
+++ b/sphinx/conf.py
@@ -43,7 +43,11 @@
 ]
 templates_path: List[str] = []
 exclude_patterns: List[str] = []
-source_suffix: Dict[str, str] = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"}
+source_suffix: Dict[str, str] = {
+    ".rst": "restructuredtext",
+    ".txt": "restructuredtext",
+    ".md": "markdown",
+}
 
 # -- Options for HTML output -------------------------------------------------
 html_theme: str = "sphinx_rtd_theme"
diff --git a/tests/test_bfs.py b/tests/test_bfs.py
index 0793126b..be5d7a7e 100644
--- a/tests/test_bfs.py
+++ b/tests/test_bfs.py
@@ -38,7 +38,13 @@ def get_sample_graph() -> Mapping[str, Sequence[str]]:
     Returns:
         graph: Python dictionary representing an adjacency list
     """
-    graph = {"1": ["2", "3", "4"], "2": ["5", "6"], "5": ["9", "10"], "4": ["7", "8"], "7": ["11", "12"]}
+    graph = {
+        "1": ["2", "3", "4"],
+        "2": ["5", "6"],
+        "5": ["9", "10"],
+        "4": ["7", "8"],
+        "7": ["11", "12"],
+    }
     return graph
 
 
@@ -61,6 +67,12 @@ def test_bfs_enumerate_paths_depth3() -> None:
 def test_bfs_enumerate_paths_depth2() -> None:
     """Graph is in adjacent list representation."""
     graph = get_sample_graph()
-    paths_ref_depth2 = [["1", "3"], ["1", "2", "6"], ["1", "4", "8"], ["1", "2", "5"], ["1", "4", "7"]]
+    paths_ref_depth2 = [
+        ["1", "3"],
+        ["1", "2", "6"],
+        ["1", "4", "8"],
+        ["1", "2", "5"],
+        ["1", "4", "7"],
+    ]
     paths = bfs_enumerate_paths(graph, "1", max_depth=2)
     assert compare_paths(paths_ref_depth2, paths)
diff --git a/tests/test_centerline_utils.py b/tests/test_centerline_utils.py
index 385d7cf4..0b1b8ff8 100644
--- a/tests/test_centerline_utils.py
+++ b/tests/test_centerline_utils.py
@@ -32,7 +32,17 @@ def temp_test_straight_centerline_to_polygon() -> None:
     polygon = centerline_to_polygon(centerline)
     # polygon wraps around with right boundary, then reversed
     # left boundary, then back to start vertex
-    gt_polygon = np.array([[-3.8, 2.0], [-3.8, 0.0], [-3.8, -2.0], [3.8, -2.0], [3.8, 0.0], [3.8, 2.0], [-3.8, 2.0]])
+    gt_polygon = np.array(
+        [
+            [-3.8, 2.0],
+            [-3.8, 0.0],
+            [-3.8, -2.0],
+            [3.8, -2.0],
+            [3.8, 0.0],
+            [3.8, 2.0],
+            [-3.8, 2.0],
+        ]
+    )
 
     assert np.array_equal(polygon, gt_polygon)
 
@@ -112,12 +122,31 @@ def test_get_nt_distance() -> None:
 
     xy = np.array([(0.0, 4.0), (3.0, 3.0), (1.0, 2.0), (4.0, 2.0), (5.0, 0.0), (7.0, 1.0)])
     centerline = np.array(
-        [(1.0, 5.0), (1.5, 4.0), (2.0, 3.0), (2.5, 2.0), (3.0, 1.0), (4.0, 1.0), (5.0, 1.0), (6.0, 1.0), (7.0, 1.0)]
+        [
+            (1.0, 5.0),
+            (1.5, 4.0),
+            (2.0, 3.0),
+            (2.5, 2.0),
+            (3.0, 1.0),
+            (4.0, 1.0),
+            (5.0, 1.0),
+            (6.0, 1.0),
+            (7.0, 1.0),
+        ]
     )
 
     nt_dist = get_nt_distance(xy, centerline)
 
-    expected_nt_dist = np.array([[1.34, 0.44], [2.68, 0.89], [2.68, 1.34], [5.47, 1.0], [6.47, 1.0], [8.47, 0.0]])
+    expected_nt_dist = np.array(
+        [
+            [1.34, 0.44],
+            [2.68, 0.89],
+            [2.68, 1.34],
+            [5.47, 1.0],
+            [6.47, 1.0],
+            [8.47, 0.0],
+        ]
+    )
     print(nt_dist, expected_nt_dist)
     np.array_equal(nt_dist, expected_nt_dist)
 
diff --git a/tests/test_dilate_utils_unit.py b/tests/test_dilate_utils_unit.py
index 5cffe99a..daa0d726 100644
--- a/tests/test_dilate_utils_unit.py
+++ b/tests/test_dilate_utils_unit.py
@@ -70,6 +70,13 @@ def test_dilate_by_l2_horizline_1point42px() -> None:
     dilated_img = dilate_by_l2(img, dilation_thresh=1.42)
     # ground truth
     dilated_img_gt = np.array(
-        [[0, 0, 0, 0, 0], [0, 1, 1, 1, 1], [0, 1, 1, 1, 1], [0, 1, 1, 1, 1], [0, 0, 0, 0, 0]], dtype=np.uint8
+        [
+            [0, 0, 0, 0, 0],
+            [0, 1, 1, 1, 1],
+            [0, 1, 1, 1, 1],
+            [0, 1, 1, 1, 1],
+            [0, 0, 0, 0, 0],
+        ],
+        dtype=np.uint8,
     )
     assert np.allclose(dilated_img, dilated_img_gt)
diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index b077d698..9588c330 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -31,7 +31,10 @@ def evaluator_identity() -> DetectionEvaluator:
     """Define an evaluator that compares a set of results to itself."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
     return DetectionEvaluator(
-        TEST_DATA_LOC / "detections_identity", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg
+        TEST_DATA_LOC / "detections_identity",
+        TEST_DATA_LOC,
+        TEST_DATA_LOC / "test_figures",
+        detection_cfg,
     )
 
 
@@ -40,7 +43,10 @@ def evaluator_assignment() -> DetectionEvaluator:
     """Define an evaluator that compares a set of results to one with an extra detection to check assignment."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
     return DetectionEvaluator(
-        TEST_DATA_LOC / "detections_assignment", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg
+        TEST_DATA_LOC / "detections_assignment",
+        TEST_DATA_LOC,
+        TEST_DATA_LOC / "test_figures",
+        detection_cfg,
     )
 
 
@@ -49,7 +55,10 @@ def evaluator() -> DetectionEvaluator:
     """Definte an evaluator that compares a set of detections with known error to the ground truth."""
     detection_cfg = DetectionCfg(dt_classes=["VEHICLE"])
     return DetectionEvaluator(
-        TEST_DATA_LOC / "detections", TEST_DATA_LOC, TEST_DATA_LOC / "test_figures", detection_cfg
+        TEST_DATA_LOC / "detections",
+        TEST_DATA_LOC,
+        TEST_DATA_LOC / "test_figures",
+        detection_cfg,
     )
 
 
diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py
index e8005d2b..c85d984c 100644
--- a/tests/test_eval_tracking.py
+++ b/tests/test_eval_tracking.py
@@ -111,7 +111,10 @@ def save_to_disk(self) -> None:
 
 
 def dump_1obj_scenario_json(
-    centers: List[Tuple[int, int, int]], yaw_angles: List[float], log_id: str, is_gt: bool
+    centers: List[Tuple[int, int, int]],
+    yaw_angles: List[float],
+    log_id: str,
+    is_gt: bool,
 ) -> None:
     """
     Egovehicle stationary (represented by `o`).
diff --git a/tests/test_frustum_clipping.py b/tests/test_frustum_clipping.py
index bdef0913..3d1182e6 100644
--- a/tests/test_frustum_clipping.py
+++ b/tests/test_frustum_clipping.py
@@ -171,7 +171,14 @@ def test_form_right_clipping_plane() -> None:
     right_plane = form_right_clipping_plane(fx, img_width)
 
     Y_OFFSET = 10  # arbitrary extent down the imager
-    right = np.array([[0, 0, 0], [img_width / 2.0, 0, fx], [0, Y_OFFSET, 0], [img_width / 2.0, Y_OFFSET, fx]])
+    right = np.array(
+        [
+            [0, 0, 0],
+            [img_width / 2.0, 0, fx],
+            [0, Y_OFFSET, 0],
+            [img_width / 2.0, Y_OFFSET, fx],
+        ]
+    )
 
     a, b, c, d = fit_plane_to_point_cloud(right)
     right_plane_gt = np.array([a, b, c, d])
@@ -191,7 +198,14 @@ def test_form_left_clipping_plane() -> None:
     left_plane = form_left_clipping_plane(fx, img_width)
 
     Y_OFFSET = 10
-    left = np.array([[0, 0, 0], [-img_width / 2.0, 0, fx], [0, Y_OFFSET, 0], [-img_width / 2.0, Y_OFFSET, fx]])
+    left = np.array(
+        [
+            [0, 0, 0],
+            [-img_width / 2.0, 0, fx],
+            [0, Y_OFFSET, 0],
+            [-img_width / 2.0, Y_OFFSET, fx],
+        ]
+    )
 
     a, b, c, d = fit_plane_to_point_cloud(left)
     left_plane_gt = -1 * np.array([a, b, c, d])
@@ -210,7 +224,13 @@ def test_form_top_clipping_plane() -> None:
     top_plane = form_top_clipping_plane(fx, img_height)
 
     img_width = 1000.0
-    top_pts = np.array([[0, 0, 0], [-img_width / 2, -img_height / 2, fx], [img_width / 2, -img_height / 2, fx]])
+    top_pts = np.array(
+        [
+            [0, 0, 0],
+            [-img_width / 2, -img_height / 2, fx],
+            [img_width / 2, -img_height / 2, fx],
+        ]
+    )
     a, b, c, d = fit_plane_to_point_cloud(top_pts)
     top_plane_gt = np.array([a, b, c, d])
 
@@ -230,7 +250,13 @@ def test_form_low_clipping_plane() -> None:
     low_plane = form_low_clipping_plane(fx, img_height)
 
     img_width = 10000
-    low_pts = np.array([[0, 0, 0], [-img_width / 2, img_height / 2, fx], [img_width / 2, img_height / 2, fx]])
+    low_pts = np.array(
+        [
+            [0, 0, 0],
+            [-img_width / 2, img_height / 2, fx],
+            [img_width / 2, img_height / 2, fx],
+        ]
+    )
     a, b, c, d = fit_plane_to_point_cloud(low_pts)
     low_plane_gt = np.array([a, b, c, d])
 
@@ -355,7 +381,16 @@ def test_cuboid_to_2d_frustum_bbox_smokescreen() -> None:
     `plot_frustum_planes_and_normals(planes, corners)` can help to do the visual check.
     """
     cuboid_verts_3d = np.array(
-        [[2, 0, 10], [-2, 0, 10], [-2, 2, 10], [2, 2, 10], [2, 0, 18], [-2, 0, 18], [-2, 2, 18], [2, 2, 18]]
+        [
+            [2, 0, 10],
+            [-2, 0, 10],
+            [-2, 2, 10],
+            [2, 2, 10],
+            [2, 0, 18],
+            [-2, 0, 18],
+            [-2, 2, 18],
+            [2, 2, 18],
+        ]
     )
     K = np.eye(3)
     # Set "focal_length_x_px_"
diff --git a/tests/test_geometry.py b/tests/test_geometry.py
index 2bb13549..4e4ec5d4 100644
--- a/tests/test_geometry.py
+++ b/tests/test_geometry.py
@@ -2,11 +2,7 @@
 
 import numpy as np
 
-from argoverse.utils.geometry import (
-    filter_point_cloud_to_polygon,
-    point_inside_polygon,
-    rotate_polygon_about_pt,
-)
+from argoverse.utils.geometry import filter_point_cloud_to_polygon, point_inside_polygon, rotate_polygon_about_pt
 
 """
 Unit tests for argoverse/utils/geometry.py
@@ -92,7 +88,16 @@ def test_rotate_polygon_about_pt_3d() -> None:
     center_pt = np.array([1.0, 1.0, 5.0])
     rotated_pts = rotate_polygon_about_pt(pts, R, center_pt)
 
-    gt_rotated_pts = np.array([[-1.5, 0, -1], [-1, 1, -1], [-2, 1, -1], [-1.5, 0, 2.3], [-1, 1, 2.3], [-2, 1, 2.3]])
+    gt_rotated_pts = np.array(
+        [
+            [-1.5, 0, -1],
+            [-1, 1, -1],
+            [-2, 1, -1],
+            [-1.5, 0, 2.3],
+            [-1, 1, 2.3],
+            [-2, 1, 2.3],
+        ]
+    )
 
     assert np.allclose(rotated_pts, gt_rotated_pts)
 
@@ -220,7 +225,20 @@ def test_filter_point_cloud_to_polygon_2d_redcross() -> None:
 
     """
     polygon = np.array(
-        [[1, 1], [1, 2], [-1, 2], [-1, 1], [-2, 1], [-2, -1], [-1, -1], [-1, -2], [1, -2], [1, -2], [2, -1], [2, 1]]
+        [
+            [1, 1],
+            [1, 2],
+            [-1, 2],
+            [-1, 1],
+            [-2, 1],
+            [-2, -1],
+            [-1, -1],
+            [-1, -2],
+            [1, -2],
+            [1, -2],
+            [2, -1],
+            [2, 1],
+        ]
     )
     point_cloud_2d = np.array([[0.9, 0.9], [1.1, 1.1], [1, 2.0], [1, 2.1], [0, 1.99]])  # in  # out  # out  # out  # in
     interior_pts = filter_point_cloud_to_polygon(polygon, point_cloud_2d)
@@ -230,7 +248,11 @@ def test_filter_point_cloud_to_polygon_2d_redcross() -> None:
 
 
 def point_inside_polygon_interior_sanity_check(
-    n_vertices: int, poly_x_pts: np.ndarray, poly_y_pts: np.ndarray, test_x: float, test_y: float
+    n_vertices: int,
+    poly_x_pts: np.ndarray,
+    poly_y_pts: np.ndarray,
+    test_x: float,
+    test_y: float,
 ) -> bool:
     """
     We use this function to verify shapely.geometry's correctness. This fn only works correctly
diff --git a/tests/test_grid_interpolation.py b/tests/test_grid_interpolation.py
index c050e2b1..3d385ab8 100644
--- a/tests/test_grid_interpolation.py
+++ b/tests/test_grid_interpolation.py
@@ -14,7 +14,14 @@ def test_interp_square_grid_nearest_3to4() -> None:
     out_dim = 4
     interp_type = "nearest"
     out_grid = interp_square_grid(grid_data, in_dim=in_dim, out_dim=out_dim, interp_type=interp_type)
-    gt_interp_grid = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 2.0, 2.0, 1.0], [1.0, 2.0, 2.0, 1.0], [1.0, 1.0, 1.0, 1.0]])
+    gt_interp_grid = np.array(
+        [
+            [1.0, 1.0, 1.0, 1.0],
+            [1.0, 2.0, 2.0, 1.0],
+            [1.0, 2.0, 2.0, 1.0],
+            [1.0, 1.0, 1.0, 1.0],
+        ]
+    )
     assert np.allclose(out_grid, gt_interp_grid)
 
 
diff --git a/tests/test_interpolate.py b/tests/test_interpolate.py
index fcdbc3e8..699c83df 100644
--- a/tests/test_interpolate.py
+++ b/tests/test_interpolate.py
@@ -4,12 +4,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from argoverse.utils.interpolate import (
-    compute_lane_width,
-    compute_mid_pivot_arc,
-    compute_midpoint_line,
-    interp_arc,
-)
+from argoverse.utils.interpolate import compute_lane_width, compute_mid_pivot_arc, compute_midpoint_line, interp_arc
 
 
 def test_compute_lane_width_straight() -> None:
diff --git a/tests/test_manhattan_search.py b/tests/test_manhattan_search.py
index fef0e2e0..7648972c 100644
--- a/tests/test_manhattan_search.py
+++ b/tests/test_manhattan_search.py
@@ -27,9 +27,18 @@ def assert_np_obj_arrs_eq(pruned_polygons: np.ndarray, gt_pruned_polygons: np.nd
 @pytest.mark.parametrize(
     "point_cloud, gt_bbox",
     [
-        (np.array([[-0.3, 0.5], [0.2, 0.1], [-0.5, 1.9]]), np.array([-0.5, 0.1, 0.2, 1.9])),
-        (np.array([[-0.3, 0.5], [-0.3, 0.5], [-0.3, 0.5]]), np.array([-0.3, 0.5, -0.3, 0.5])),
-        (np.array([[-0.3, 0.5, 50.1], [0.2, 0.1, -100.3], [-0.5, 1.9, -0.01]]), np.array([-0.5, 0.1, 0.2, 1.9])),
+        (
+            np.array([[-0.3, 0.5], [0.2, 0.1], [-0.5, 1.9]]),
+            np.array([-0.5, 0.1, 0.2, 1.9]),
+        ),
+        (
+            np.array([[-0.3, 0.5], [-0.3, 0.5], [-0.3, 0.5]]),
+            np.array([-0.3, 0.5, -0.3, 0.5]),
+        ),
+        (
+            np.array([[-0.3, 0.5, 50.1], [0.2, 0.1, -100.3], [-0.5, 1.9, -0.01]]),
+            np.array([-0.5, 0.1, 0.2, 1.9]),
+        ),
     ],
 )  # type: ignore
 def test_compute_point_cloud_bbox_2d(point_cloud: np.ndarray, gt_bbox: np.ndarray) -> None:
@@ -52,7 +61,13 @@ def polygons_and_gt_bboxes() -> Tuple[List[np.ndarray], List[np.ndarray]]:
     gt_poly_3_bbox = np.array([0.5, 2.5, 1.5, 3.5])
     gt_poly_4_bbox = np.array([-2.5, -1.5, 1.5, 1.75])
     gt_poly_5_bbox = np.array([1.5, 0.5, 1.5, 1.1])
-    gt_poly_bboxes = [gt_poly_1_bbox, gt_poly_2_bbox, gt_poly_3_bbox, gt_poly_4_bbox, gt_poly_5_bbox]
+    gt_poly_bboxes = [
+        gt_poly_1_bbox,
+        gt_poly_2_bbox,
+        gt_poly_3_bbox,
+        gt_poly_4_bbox,
+        gt_poly_5_bbox,
+    ]
 
     return polygons, gt_poly_bboxes
 
@@ -79,7 +94,11 @@ def test_compute_polygon_bboxes(polygons_and_gt_bboxes: Tuple[List[np.ndarray],
 
 @pytest.mark.parametrize(
     "query_pt, query_search_range_manhattan, gt_indices",
-    [(np.array([-0.5, 1.5]), 0.5, [0, 1, 3]), (np.array([-0.5, 1.5]), 0.499, [0, 3]), (np.array([0, 2]), 0.24, [])],
+    [
+        (np.array([-0.5, 1.5]), 0.5, [0, 1, 3]),
+        (np.array([-0.5, 1.5]), 0.499, [0, 3]),
+        (np.array([0, 2]), 0.24, []),
+    ],
 )  # type: ignore
 def test_prune_polygons_manhattan_dist_find_nearby(
     query_pt: np.ndarray,
diff --git a/tests/test_pkl_utils.py b/tests/test_pkl_utils.py
index ce74c6c9..d65cc9ba 100644
--- a/tests/test_pkl_utils.py
+++ b/tests/test_pkl_utils.py
@@ -36,7 +36,12 @@ def test_save_pickle_from_disk() -> None:
     {'a': 1, 'b':'2', 'c':[9,8,7,6,5,'d','c','b','a'], 'd': np.array([True,False,True]) }
     """
     pkl_fpath = _TEST_DIR / "test_data/pkl_test_file.pkl"
-    intended_dict = {"a": 1, "b": "2", "c": [9, 8, 7, 6, 5, "d", "c", "b", "a"], "d": np.array([True, False, True])}
+    intended_dict = {
+        "a": 1,
+        "b": "2",
+        "c": [9, 8, 7, 6, 5, "d", "c", "b", "a"],
+        "d": np.array([True, False, True]),
+    }
     save_pkl_dictionary(pkl_fpath, intended_dict)
 
     with open(pkl_fpath, "rb") as f:
@@ -52,7 +57,12 @@ def test_load_pickle_from_disk() -> None:
 
     We demonstrate that we can load Numpy arrays and lists from Pickle files.
     """
-    gt_dict = {"a": 1, "b": "2", "c": [9, 8, 7, 6, 5, "d", "c", "b", "a"], "d": np.array([True, False, True])}
+    gt_dict = {
+        "a": 1,
+        "b": "2",
+        "c": [9, 8, 7, 6, 5, "d", "c", "b", "a"],
+        "d": np.array([True, False, True]),
+    }
     pkl_fpath = _TEST_DIR / "test_data/pkl_test_file.pkl"
     loaded_pkl_dict = load_pkl_dictionary(pkl_fpath)
     dictionaries_are_equal(gt_dict, loaded_pkl_dict)
diff --git a/tests/test_se2.py b/tests/test_se2.py
index f95ac25d..bb06068d 100644
--- a/tests/test_se2.py
+++ b/tests/test_se2.py
@@ -32,7 +32,11 @@ def test_SE2_constructor() -> None:
     cos_theta = np.cos(theta)
     sin_theta = np.sin(theta)
     T_mat_gt = np.array(
-        [[cos_theta, -sin_theta, translation_vector[0]], [sin_theta, cos_theta, translation_vector[1]], [0, 0, 1.0]]
+        [
+            [cos_theta, -sin_theta, translation_vector[0]],
+            [sin_theta, cos_theta, translation_vector[1]],
+            [0, 0, 1.0],
+        ]
     )
 
     assert np.allclose(dst_se2_src.rotation, rotation_matrix)
diff --git a/tests/test_se3.py b/tests/test_se3.py
index 8c42aeb5..ece25065 100644
--- a/tests/test_se3.py
+++ b/tests/test_se3.py
@@ -147,7 +147,12 @@ def test_SE3_chaining_transforms() -> None:
 
     pts = np.array([[1.0, 0.0, 4.0], [1.0, 0.0, 3.0]])
     transformed_pts = fr2_se3_fr0.transform_point_cloud(pts.copy())
-    gt_transformed_pts = np.array([[-np.sqrt(2) / 2, -np.sqrt(2) / 2, 4.0], [-np.sqrt(2) / 2, -np.sqrt(2) / 2, 3.0]])
+    gt_transformed_pts = np.array(
+        [
+            [-np.sqrt(2) / 2, -np.sqrt(2) / 2, 4.0],
+            [-np.sqrt(2) / 2, -np.sqrt(2) / 2, 3.0],
+        ]
+    )
 
     assert np.allclose(transformed_pts, gt_transformed_pts)
 
diff --git a/tests/test_simple_track_dataloader.py b/tests/test_simple_track_dataloader.py
index f6f94130..4aa6f087 100644
--- a/tests/test_simple_track_dataloader.py
+++ b/tests/test_simple_track_dataloader.py
@@ -5,9 +5,7 @@
 
 import pytest
 
-from argoverse.data_loading.simple_track_dataloader import (
-    SimpleArgoverseTrackingDataLoader,
-)
+from argoverse.data_loading.simple_track_dataloader import SimpleArgoverseTrackingDataLoader
 
 _TEST_DATA = pathlib.Path(__file__).parent / "test_data" / "tracking"
 _LOG_ID = "1"
@@ -22,12 +20,16 @@ def test_get_city_name(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
     assert data_loader.get_city_name(_LOG_ID) == "PIT"
 
 
-def test_get_log_calibration_data(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_log_calibration_data(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     # Just check that it doesn't raise.
     assert data_loader.get_log_calibration_data(_LOG_ID)
 
 
-def test_get_city_to_egovehicle_se3(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_city_to_egovehicle_se3(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     assert data_loader.get_city_to_egovehicle_se3(_LOG_ID, 0) is not None
     assert data_loader.get_city_to_egovehicle_se3(_LOG_ID, 100) is None
 
@@ -37,17 +39,23 @@ def test_get_closest_im_fpath(data_loader: SimpleArgoverseTrackingDataLoader) ->
     assert data_loader.get_closest_im_fpath(_LOG_ID, "nonexisting_camera_name", 0) is None
 
 
-def test_get_ordered_log_ply_fpaths(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_ordered_log_ply_fpaths(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     # Test data doesn't have cameras so we cannot currently test this if we
     assert len(data_loader.get_ordered_log_ply_fpaths(_LOG_ID)) == 3
 
 
-def test_get_labels_at_lidar_timestamp(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_labels_at_lidar_timestamp(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     assert data_loader.get_labels_at_lidar_timestamp(_LOG_ID, 0) is not None
     assert data_loader.get_labels_at_lidar_timestamp(_LOG_ID, 100) is None
 
 
-def test_get_closest_im_fpath_exists(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_closest_im_fpath_exists(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     # Test data does have ring front cameras at timestamps 0,1,2,3. Compare with ground truth (gt)
     im_fpath = data_loader.get_closest_im_fpath(_LOG_ID, "ring_front_right", 2)
     assert im_fpath is not None
@@ -56,7 +64,9 @@ def test_get_closest_im_fpath_exists(data_loader: SimpleArgoverseTrackingDataLoa
     assert "/".join(im_fpath.split("/")[-5:]) == gt_im_fpath
 
 
-def test_get_closest_lidar_fpath_found_match(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_closest_lidar_fpath_found_match(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     """ Just barely within 51 ms allowed buffer"""
     cam_timestamp = int(50 * 1e6)
     ply_fpath = data_loader.get_closest_lidar_fpath(_LOG_ID, cam_timestamp)
@@ -66,7 +76,9 @@ def test_get_closest_lidar_fpath_found_match(data_loader: SimpleArgoverseTrackin
     assert "/".join(ply_fpath.split("/")[-5:]) == gt_ply_fpath
 
 
-def test_get_closest_lidar_fpath_no_match(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_closest_lidar_fpath_no_match(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     """LiDAR rotates at 10 Hz (sensor message per 100 ms). Test if camera measurement
     just barely outside 51 ms allowed buffer. Max LiDAR timestamp in log is 2.
     51 ms, not 50 ms, is allowed to give time for occasional delay.
@@ -78,7 +90,9 @@ def test_get_closest_lidar_fpath_no_match(data_loader: SimpleArgoverseTrackingDa
     assert ply_fpath is None
 
 
-def test_get_ordered_log_cam_fpaths(data_loader: SimpleArgoverseTrackingDataLoader) -> None:
+def test_get_ordered_log_cam_fpaths(
+    data_loader: SimpleArgoverseTrackingDataLoader,
+) -> None:
     """ Make sure all images for one camera in one log are returned in correct order. """
     camera_name = "ring_rear_right"
     cam_img_fpaths = data_loader.get_ordered_log_cam_fpaths(_LOG_ID, camera_name)
diff --git a/tests/test_transform.py b/tests/test_transform.py
index 3f0bc01b..891e35ec 100644
--- a/tests/test_transform.py
+++ b/tests/test_transform.py
@@ -57,7 +57,14 @@ def test_quat2rotmat_1() -> None:
 
 def test_quat2rotmat_2() -> None:
     """Test receiving a quaternion in (w, x, y, z) from a camera extrinsic matrix."""
-    q = np.array([0.4962730586309743, -0.503110985154011, 0.4964713836540661, -0.5040918101963521])
+    q = np.array(
+        [
+            0.4962730586309743,
+            -0.503110985154011,
+            0.4964713836540661,
+            -0.5040918101963521,
+        ]
+    )
     R = quat2rotmat(q)
     assert np.allclose(R, quat2rotmat_numpy(q))
 
@@ -73,7 +80,14 @@ def test_quat2rotmat_2() -> None:
 
 def test_quat2rotmat_3() -> None:
     """Test receiving a quaternion in (w, x, y, z) from a camera extrinsic matrix."""
-    q = np.array([0.6115111374269877, -0.6173269265351116, -0.3480540121107544, 0.3518806604959585])
+    q = np.array(
+        [
+            0.6115111374269877,
+            -0.6173269265351116,
+            -0.3480540121107544,
+            0.3518806604959585,
+        ]
+    )
     R = quat2rotmat(q)
     assert np.allclose(R, quat2rotmat_numpy(q))
 
@@ -89,7 +103,14 @@ def test_quat2rotmat_3() -> None:
 
 def test_quat2rotmat_4() -> None:
     """Test receiving a quaternion in (w, x, y, z) from an object trajectory."""
-    q = np.array([0.0036672729619914197, -1.3748614058859026e-05, -0.00023389080405946338, 0.9999932480847505])
+    q = np.array(
+        [
+            0.0036672729619914197,
+            -1.3748614058859026e-05,
+            -0.00023389080405946338,
+            0.9999932480847505,
+        ]
+    )
     R = quat2rotmat(q)
     assert np.allclose(R, quat2rotmat_numpy(q))
 
@@ -105,7 +126,14 @@ def test_quat2rotmat_4() -> None:
 
 def test_quat2rotmat_5() -> None:
     """Test receiving a quaternion in (w, x, y, z) from an object trajectory."""
-    q = np.array([0.9998886199825181, -0.002544078377693514, -0.0028621717588219564, -0.01442509159370476])
+    q = np.array(
+        [
+            0.9998886199825181,
+            -0.002544078377693514,
+            -0.0028621717588219564,
+            -0.01442509159370476,
+        ]
+    )
     R = quat2rotmat(q)
     assert np.allclose(R, quat2rotmat_numpy(q))
 

From 96b83bd19163aed48d42b6b2ca43c277e46a1587 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 1 Oct 2020 19:52:08 -0400
Subject: [PATCH 104/113] Ensure 120 length isort.

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index d4bad340..43273d86 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
     rev: 687aecc
     hooks:
     - id: isort
-      args: ["--profile", "black"]
+      args: ["--profile", "black", "-l", "120"]
 -   repo: https://github.com/psf/black
     rev: 172c0a7
     hooks:

From 64ecce33946ee3f3b32077969cabe9d25ce7cd02 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Fri, 2 Oct 2020 10:52:01 -0400
Subject: [PATCH 105/113] Added units tests and stubs.

---
 tests/test_eval_detection.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index 9588c330..11024871 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -18,6 +18,7 @@
     compute_affinity_matrix,
     dist_fn,
     iou_aligned_3d,
+    wrap_angle,
 )
 from argoverse.evaluation.eval_detection import DetectionEvaluator
 from argoverse.utils.transform import quat_scipy2argo_vectorized
@@ -173,6 +174,38 @@ def test_orientation_eighth_angles() -> None:
         assert np.isclose(dist_fn(gts, dts, DistFnType.ORIENTATION), expected_result)
 
 
+def test_wrap_angle() -> None:
+    theta: np.ndarray = np.array([-3 * np.pi / 2])
+
+    expected_result: float = np.array([np.pi / 2])
+    assert wrap_angle(theta) == expected_result
+
+
+# TODO Stub
+def test_accumulate() -> None:
+    pass
+
+
+# TODO Stub
+def test_assign() -> None:
+    pass
+
+
+# TODO Stub
+def test_filter_instances() -> None:
+    pass
+
+
+# TODO Stub
+def test_interp() -> None:
+    pass
+
+
+# TODO Stub
+def test_plot() -> None:
+    pass
+
+
 def test_iou_aligned_3d() -> None:
     """Initialize a detection and a ground truth label with only shape
     parameters (only shape parameters due to alignment assumption).

From c778106142550a1c132d5778a74096d7f39af056 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Mon, 5 Oct 2020 06:57:50 -0400
Subject: [PATCH 106/113] Added new tests + updated docstrings.

---
 argoverse/evaluation/detection_utils.py | 15 ++++++--
 tests/test_eval_detection.py            | 51 +++++++++++++++++++++----
 2 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index 79313e50..fe27ed78 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -270,7 +270,9 @@ def rank(dts: List[ObjectLabelRecord]) -> Tuple[np.ndarray, np.ndarray]:
 
 
 def interp(prec: np.ndarray, method: InterpType = InterpType.ALL) -> np.ndarray:
-    """Interpolate the precision over all recall levels.
+    """Interpolate the precision over all recall levels. See equation 2 in
+    http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.167.6629&rep=rep1&type=pdf
+    for more information.
 
     Args:
         prec: Precision at all recall levels (N,).
@@ -393,7 +395,7 @@ def wrap_angle(angles: np.ndarray, period: float = np.pi) -> np.ndarray:
         the inverse of `np.unwrap`.
 
     Returns:
-        The angles (in radians) mapped to the interval [0, π).
+        Angles (in radians) mapped to the interval [0, π).
     """
 
     # Map angles to [0, ∞].
@@ -411,7 +413,7 @@ def wrap_angle(angles: np.ndarray, period: float = np.pi) -> np.ndarray:
     return angles
 
 
-def plot(rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str, figs_fpath: Path) -> None:
+def plot(rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str, figs_fpath: Path) -> Path:
     """Plot and save the precision recall curve.
 
     Args:
@@ -419,10 +421,15 @@ def plot(rec_interp: np.ndarray, prec_interp: np.ndarray, cls_name: str, figs_fp
         prec_interp: Interpolated precision data of shape (N,).
         cls_name: Class name.
         figs_fpath: Path to the folder which will contain the output figures.
+    Returns:
+        dst_fpath: Plot file path.
     """
     plt.plot(rec_interp, prec_interp)
     plt.title("PR Curve")
     plt.xlabel("Recall")
     plt.ylabel("Precision")
-    plt.savefig(f"{figs_fpath}/{cls_name}.png")
+
+    dst_fpath = Path(f"{figs_fpath}/{cls_name}.png")
+    plt.savefig(dst_fpath)
     plt.close()
+    return dst_fpath
diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index 11024871..4e44637e 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -2,7 +2,7 @@
 """Detection evaluation unit tests"""
 
 import logging
-import pathlib
+from pathlib import Path
 from typing import List
 
 import numpy as np
@@ -15,15 +15,19 @@
     AffFnType,
     DetectionCfg,
     DistFnType,
+    FilterMetric,
     compute_affinity_matrix,
     dist_fn,
+    filter_instances,
+    interp,
     iou_aligned_3d,
+    plot,
     wrap_angle,
 )
 from argoverse.evaluation.eval_detection import DetectionEvaluator
 from argoverse.utils.transform import quat_scipy2argo_vectorized
 
-TEST_DATA_LOC = pathlib.Path(__file__).parent.parent / "tests" / "test_data" / "detection"
+TEST_DATA_LOC = Path(__file__).parent.parent / "tests" / "test_data" / "detection"
 logging.getLogger("matplotlib.font_manager").disabled = True
 
 
@@ -191,19 +195,50 @@ def test_assign() -> None:
     pass
 
 
-# TODO Stub
 def test_filter_instances() -> None:
-    pass
+    """Generate 100 different detections and filter them based on Euclidean distance."""
+    dts: List[ObjectLabelRecord] = [
+        ObjectLabelRecord(
+            translation=[i, i, 0],
+            quaternion=np.array([0, 0, 0, 0]),
+            length=5.0,
+            width=2.0,
+            height=3.0,
+            occlusion=0,
+            label_class="VEHICLE",
+        )
+        for i in range(100)
+    ]
+
+    target_class_name: str = "VEHICLE"
+    filter_metric: FilterMetric = FilterMetric.EUCLIDEAN
+    max_detection_range: float = 100.0
+
+    expected_result: int = 71
+    assert len(filter_instances(dts, target_class_name, filter_metric, max_detection_range)) == expected_result
 
 
-# TODO Stub
 def test_interp() -> None:
-    pass
+    """Test non-decreasing `interpolation` constraint enforced on precision results.
+    See equation 2 in http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.167.6629&rep=rep1&type=pdf
+    for more information."""
+    prec: np.ndarray = np.array([1.0, 0.5, 0.33, 0.5])
+
+    expected_result: np.ndarray = np.array([1.0, 0.5, 0.5, 0.5])
+    assert (interp(prec) == expected_result).all()
 
 
-# TODO Stub
 def test_plot() -> None:
-    pass
+    """Test plotting functionality (i.e., plots are written to specified file)."""
+    prec_interp: np.ndarray = np.array([1.0, 0.5, 0.25, 0.125])
+    rec_interp: np.ndarray = np.array([0.25, 0.5, 0.75, 1.0])
+    cls_name: str = "VEHICLE"
+    figs_fpath: Path = Path("/tmp/figs")
+    if not figs_fpath.is_dir():
+        figs_fpath.mkdir(parents=True, exist_ok=True)
+
+    expected_result: Path = Path(figs_fpath / (cls_name + ".png"))
+    assert plot(rec_interp, prec_interp, cls_name, figs_fpath) == expected_result
 
 
 def test_iou_aligned_3d() -> None:

From 3bf475c127b72770474b45eda09c2d2ed678572b Mon Sep 17 00:00:00 2001
From: Sean 
Date: Mon, 5 Oct 2020 16:33:12 -0400
Subject: [PATCH 107/113] add assign and accumulate tests, plus other small
 fixes

---
 argoverse/evaluation/detection_utils.py |  1 -
 tests/test_eval_detection.py            | 89 +++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection_utils.py
index fe27ed78..6fa01771 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection_utils.py
@@ -166,7 +166,6 @@ def assign(dts: np.ndarray, gts: np.ndarray, cfg: DetectionCfg) -> np.ndarray:
     Returns:
         metrics: Matrix of true/false positive concatenated with true positive errors (N, K + S) where K is the number
             of true positive thresholds used for AP computation and S is the number of true positive errors.
-        scores: Corresponding scores for the true positives/false positives (N,).
     """
 
     # Ensure the number of boxes considered per class is at most `MAX_NUM_BOXES`.
diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index 4e44637e..0db6f124 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -16,6 +16,8 @@
     DetectionCfg,
     DistFnType,
     FilterMetric,
+    accumulate,
+    assign,
     compute_affinity_matrix,
     dist_fn,
     filter_instances,
@@ -91,7 +93,7 @@ def test_affinity_center() -> None:
     """
     dts: List[ObjectLabelRecord] = [
         ObjectLabelRecord(
-            quaternion=np.array([0, 0, 0, 0]),
+            quaternion=np.array([1, 0, 0, 0]),
             translation=np.array([0, 0, 0]),
             length=5.0,
             width=5.0,
@@ -101,7 +103,7 @@ def test_affinity_center() -> None:
     ]
     gts: List[ObjectLabelRecord] = [
         ObjectLabelRecord(
-            quaternion=np.array([0, 0, 0, 0]),
+            quaternion=np.array([1, 0, 0, 0]),
             translation=np.array([3, 4, 0]),
             length=5.0,
             width=5.0,
@@ -185,14 +187,89 @@ def test_wrap_angle() -> None:
     assert wrap_angle(theta) == expected_result
 
 
-# TODO Stub
 def test_accumulate() -> None:
-    pass
+    """Verify that the accumulate function matches known output for a self-comparison."""
+    cfg = DetectionCfg()
+    # compare a set of labels to itself
+    cls_to_accum, cls_to_ninst = accumulate(
+        TEST_DATA_LOC / "detections",
+        TEST_DATA_LOC / "detections/1/per_sweep_annotations_amodal/tracked_object_labels_0.json",
+        cfg,
+    )
+    # ensure the detections match at all thresholds, have 0 TP errors, and have AP = 1
+    assert (
+        cls_to_accum["VEHICLE"]
+        == np.array([[1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0]])
+    ).all()
+    assert cls_to_ninst["VEHICLE"] == 2  # there are 2 vehicle labels in this file
+    assert sum(cls_to_ninst.values()) == 2  # and no other labels
 
 
-# TODO Stub
 def test_assign() -> None:
-    pass
+    """Verify that the assign functions as expected by checking ATE of assigned detections against known distance."""
+    cfg = DetectionCfg()
+    dts: np.ndarray = np.array(
+        [
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([0, 0, 0]),
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([10, 10, 10]),
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([20, 20, 20]),
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+        ]
+    )
+    gts: np.ndarray = np.array(
+        [
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([-10, -10, -10]),
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([0.1, 0, 0]),  # off by 0.1
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+            ObjectLabelRecord(
+                quaternion=np.array([1, 0, 0, 0]),
+                translation=np.array([10.1, 10, 10]),  # off by 0.1
+                length=5.0,
+                width=5.0,
+                height=5.0,
+                occlusion=0,
+            ),
+        ]
+    )
+    metrics = assign(dts, gts, cfg)
+    # if these assign correctly, we should get an ATE of 0.1 for the first two
+    expected_result: float = 0.1
+    assert np.isclose(metrics[0, 4], expected_result)
+    assert np.isclose(metrics[1, 4], expected_result)
+    assert np.isnan(metrics[2, 5])
 
 
 def test_filter_instances() -> None:

From e1adfa67054df50e37be7a0515053eef06007c2a Mon Sep 17 00:00:00 2001
From: Sean 
Date: Mon, 5 Oct 2020 17:12:32 -0400
Subject: [PATCH 108/113] slight clarification of some unit tests

---
 tests/test_eval_detection.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index 0db6f124..b5e80be4 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -197,9 +197,18 @@ def test_accumulate() -> None:
         cfg,
     )
     # ensure the detections match at all thresholds, have 0 TP errors, and have AP = 1
+    expected_ATE = 0.0
+    expected_ASE = 0.0
+    expected_AOE = 0.0
+    expected_AP = 1.0
     assert (
         cls_to_accum["VEHICLE"]
-        == np.array([[1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0]])
+        == np.array(
+            [
+                [1.0, 1.0, 1.0, 1.0, expected_ATE, expected_ASE, expected_AOE, expected_AP],
+                [1.0, 1.0, 1.0, 1.0, expected_ATE, expected_ASE, expected_AOE, expected_AP],
+            ]
+        )
     ).all()
     assert cls_to_ninst["VEHICLE"] == 2  # there are 2 vehicle labels in this file
     assert sum(cls_to_ninst.values()) == 2  # and no other labels
@@ -267,9 +276,10 @@ def test_assign() -> None:
     metrics = assign(dts, gts, cfg)
     # if these assign correctly, we should get an ATE of 0.1 for the first two
     expected_result: float = 0.1
-    assert np.isclose(metrics[0, 4], expected_result)
-    assert np.isclose(metrics[1, 4], expected_result)
-    assert np.isnan(metrics[2, 5])
+    ATE_COL_IDX = 4
+    assert np.isclose(metrics[0, ATE_COL_IDX], expected_result)  # instance 0
+    assert np.isclose(metrics[1, ATE_COL_IDX], expected_result)  # instance 1
+    assert np.isnan(metrics[2, ATE_COL_IDX])  # instance 32
 
 
 def test_filter_instances() -> None:

From 24d1c5d8ebd435321464a0db95c38e4541184def Mon Sep 17 00:00:00 2001
From: Ben Wilson 
Date: Mon, 5 Oct 2020 22:15:43 +0000
Subject: [PATCH 109/113] Removed unused import.

---
 argoverse/evaluation/eval_detection.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/eval_detection.py
index f4f96311..98cf244e 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/eval_detection.py
@@ -68,7 +68,6 @@
 
 import numpy as np
 import pandas as pd
-from pandas.core import frame
 
 from argoverse.evaluation.detection_utils import DetectionCfg, accumulate, calc_ap, plot
 

From 7301fd1dfa0bbd0252dfd4d7495c1b48b17f3f87 Mon Sep 17 00:00:00 2001
From: Ubuntu 
Date: Wed, 7 Oct 2020 16:39:12 +0000
Subject: [PATCH 110/113] Restructured fpaths.

---
 argoverse/evaluation/detection/constants.py   | 27 +++++++++++++++
 .../{eval_detection.py => detection/eval.py}  | 11 ++----
 .../utils.py}                                 | 34 ++++++-------------
 argoverse/evaluation/eval_tracking.py         |  3 +-
 tests/test_eval_detection.py                  |  4 +--
 tests/test_eval_tracking.py                   |  2 +-
 6 files changed, 43 insertions(+), 38 deletions(-)
 create mode 100644 argoverse/evaluation/detection/constants.py
 rename argoverse/evaluation/{eval_detection.py => detection/eval.py} (97%)
 rename argoverse/evaluation/{detection_utils.py => detection/utils.py} (96%)

diff --git a/argoverse/evaluation/detection/constants.py b/argoverse/evaluation/detection/constants.py
new file mode 100644
index 00000000..3e872e99
--- /dev/null
+++ b/argoverse/evaluation/detection/constants.py
@@ -0,0 +1,27 @@
+from typing import List
+
+import numpy as np
+
+COMPETITION_CLASSES: List[str] = ["VEHICLE", "PEDESTRIAN"]
+
+TP_ERROR_NAMES: List[str] = ["ATE", "ASE", "AOE"]
+N_TP_ERRORS: int = len(TP_ERROR_NAMES)
+
+STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"]
+
+MAX_SCALE_ERROR: float = 1.0
+MAX_YAW_ERROR: float = np.pi
+
+# Higher is better.
+MIN_AP: float = 0.0
+MIN_CDS: float = 0.0
+
+# Lower is better.
+MAX_NORMALIZED_ATE: float = 1.0
+MAX_NORMALIZED_ASE: float = 1.0
+MAX_NORMALIZED_AOE: float = 1.0
+
+# Max number of boxes considered per class per scene.
+MAX_NUM_BOXES: int = 500
+
+SIGNIFICANT_DIGITS: float = 3
diff --git a/argoverse/evaluation/eval_detection.py b/argoverse/evaluation/detection/eval.py
similarity index 97%
rename from argoverse/evaluation/eval_detection.py
rename to argoverse/evaluation/detection/eval.py
index 98cf244e..22d2bf49 100644
--- a/argoverse/evaluation/eval_detection.py
+++ b/argoverse/evaluation/detection/eval.py
@@ -69,19 +69,12 @@
 import numpy as np
 import pandas as pd
 
-from argoverse.evaluation.detection_utils import DetectionCfg, accumulate, calc_ap, plot
+from argoverse.evaluation.detection.constants import N_TP_ERRORS, SIGNIFICANT_DIGITS, STATISTIC_NAMES, np
+from argoverse.evaluation.detection.utils import DetectionCfg, accumulate, calc_ap, plot
 
 logger = logging.getLogger(__name__)
 
 
-TP_ERROR_NAMES: List[str] = ["ATE", "ASE", "AOE"]
-N_TP_ERRORS: int = len(TP_ERROR_NAMES)
-
-STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"]
-
-SIGNIFICANT_DIGITS: float = 3
-
-
 class DetectionEvaluator(NamedTuple):
     """Instantiates a DetectionEvaluator object for evaluation.
 
diff --git a/argoverse/evaluation/detection_utils.py b/argoverse/evaluation/detection/utils.py
similarity index 96%
rename from argoverse/evaluation/detection_utils.py
rename to argoverse/evaluation/detection/utils.py
index 6fa01771..fd727ce3 100644
--- a/argoverse/evaluation/detection_utils.py
+++ b/argoverse/evaluation/detection/utils.py
@@ -23,38 +23,24 @@
 
 from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT
 from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label
+from argoverse.evaluation.detection.constants import (
+    MAX_NORMALIZED_AOE,
+    MAX_NORMALIZED_ASE,
+    MAX_NUM_BOXES,
+    MAX_SCALE_ERROR,
+    MAX_YAW_ERROR,
+    MIN_AP,
+    MIN_CDS,
+    N_TP_ERRORS,
+)
 from argoverse.utils.transform import quat_argo2scipy_vectorized
 
 matplotlib.use("Agg")  # isort:skip
 import matplotlib.pyplot as plt  # isort:skip  # noqa: E402
 
-
 logger = logging.getLogger(__name__)
 
 
-TP_ERROR_NAMES: List[str] = ["ATE", "ASE", "AOE"]
-N_TP_ERRORS: int = len(TP_ERROR_NAMES)
-
-STATISTIC_NAMES: List[str] = ["AP"] + TP_ERROR_NAMES + ["CDS"]
-
-MAX_SCALE_ERROR: float = 1.0
-MAX_YAW_ERROR: float = np.pi
-
-# Higher is better.
-MIN_AP: float = 0.0
-MIN_CDS: float = 0.0
-
-# Lower is better.
-MAX_NORMALIZED_ATE: float = 1.0
-MAX_NORMALIZED_ASE: float = 1.0
-MAX_NORMALIZED_AOE: float = 1.0
-
-# Max number of boxes considered per class per scene.
-MAX_NUM_BOXES: int = 500
-
-SIGNIFICANT_DIGITS: float = 3
-
-
 class AffFnType(Enum):
     CENTER = auto()
 
diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py
index 187c2fc0..14fb0523 100644
--- a/argoverse/evaluation/eval_tracking.py
+++ b/argoverse/evaluation/eval_tracking.py
@@ -12,11 +12,10 @@
 import numpy as np
 from shapely.geometry.polygon import Polygon
 
+from argoverse.evaluation.detection.utils import wrap_angle
 from argoverse.evaluation.eval_utils import label_to_bbox
 from argoverse.utils.json_utils import read_json_file
 
-from .detection_utils import wrap_angle
-
 mh = mm.metrics.create()
 logger = logging.getLogger(__name__)
 
diff --git a/tests/test_eval_detection.py b/tests/test_eval_detection.py
index b5e80be4..a2bad6a2 100644
--- a/tests/test_eval_detection.py
+++ b/tests/test_eval_detection.py
@@ -11,7 +11,8 @@
 from scipy.spatial.transform import Rotation as R
 
 from argoverse.data_loading.object_label_record import ObjectLabelRecord
-from argoverse.evaluation.detection_utils import (
+from argoverse.evaluation.detection.eval import DetectionEvaluator
+from argoverse.evaluation.detection.utils import (
     AffFnType,
     DetectionCfg,
     DistFnType,
@@ -26,7 +27,6 @@
     plot,
     wrap_angle,
 )
-from argoverse.evaluation.eval_detection import DetectionEvaluator
 from argoverse.utils.transform import quat_scipy2argo_vectorized
 
 TEST_DATA_LOC = Path(__file__).parent.parent / "tests" / "test_data" / "detection"
diff --git a/tests/test_eval_tracking.py b/tests/test_eval_tracking.py
index c85d984c..f83417d2 100644
--- a/tests/test_eval_tracking.py
+++ b/tests/test_eval_tracking.py
@@ -9,7 +9,7 @@
 import numpy as np
 from scipy.spatial.transform import Rotation
 
-from argoverse.evaluation.detection_utils import wrap_angle
+from argoverse.evaluation.detection.utils import wrap_angle
 from argoverse.evaluation.eval_tracking import eval_tracks
 from argoverse.utils.json_utils import save_json_dict
 

From 95845eda3052110627deff1c12cc900a0783f20d Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 7 Oct 2020 16:42:23 +0000
Subject: [PATCH 111/113] Small fixes.

---
 argoverse/evaluation/detection/eval.py  | 2 +-
 argoverse/evaluation/detection/utils.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/argoverse/evaluation/detection/eval.py b/argoverse/evaluation/detection/eval.py
index 22d2bf49..b3364c49 100644
--- a/argoverse/evaluation/detection/eval.py
+++ b/argoverse/evaluation/detection/eval.py
@@ -69,7 +69,7 @@
 import numpy as np
 import pandas as pd
 
-from argoverse.evaluation.detection.constants import N_TP_ERRORS, SIGNIFICANT_DIGITS, STATISTIC_NAMES, np
+from argoverse.evaluation.detection.constants import N_TP_ERRORS, SIGNIFICANT_DIGITS, STATISTIC_NAMES
 from argoverse.evaluation.detection.utils import DetectionCfg, accumulate, calc_ap, plot
 
 logger = logging.getLogger(__name__)
diff --git a/argoverse/evaluation/detection/utils.py b/argoverse/evaluation/detection/utils.py
index fd727ce3..b27defa8 100644
--- a/argoverse/evaluation/detection/utils.py
+++ b/argoverse/evaluation/detection/utils.py
@@ -24,6 +24,7 @@
 from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT
 from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label
 from argoverse.evaluation.detection.constants import (
+    COMPETITION_CLASSES,
     MAX_NORMALIZED_AOE,
     MAX_NORMALIZED_ASE,
     MAX_NUM_BOXES,
@@ -80,7 +81,7 @@ class DetectionCfg(NamedTuple):
     affinity_fn_type: AffFnType = AffFnType.CENTER
     n_rec_samples: int = 101
     tp_thresh: float = 2.0  # Meters
-    dt_classes: List[str] = list(OBJ_CLASS_MAPPING_DICT.keys())
+    dt_classes: List[str] = COMPETITION_CLASSES
     dt_metric: FilterMetric = FilterMetric.EUCLIDEAN
     max_dt_range: float = 100.0  # Meters
     save_figs: bool = False

From de65f3e52a25d86a1cc8f39fed033b9eb83d1795 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Wed, 7 Oct 2020 16:44:56 +0000
Subject: [PATCH 112/113] Removed unused import.

---
 argoverse/evaluation/detection/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/argoverse/evaluation/detection/utils.py b/argoverse/evaluation/detection/utils.py
index b27defa8..3814dbf6 100644
--- a/argoverse/evaluation/detection/utils.py
+++ b/argoverse/evaluation/detection/utils.py
@@ -21,7 +21,6 @@
 from scipy.spatial.distance import cdist
 from scipy.spatial.transform import Rotation as R
 
-from argoverse.data_loading.object_classes import OBJ_CLASS_MAPPING_DICT
 from argoverse.data_loading.object_label_record import ObjectLabelRecord, read_label
 from argoverse.evaluation.detection.constants import (
     COMPETITION_CLASSES,

From f77119cea068b40734ecba31a20e1a406e1cb499 Mon Sep 17 00:00:00 2001
From: Benjamin Wilson 
Date: Thu, 8 Oct 2020 16:55:05 +0000
Subject: [PATCH 113/113] Uncommented unit test.

---
 tests/test_visualization_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_visualization_utils.py b/tests/test_visualization_utils.py
index ffe1a972..494a24cc 100644
--- a/tests/test_visualization_utils.py
+++ b/tests/test_visualization_utils.py
@@ -44,8 +44,8 @@ def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoad
     visualization_utils.draw_point_cloud_trajectory(axes, "title!", data_loader, 0, [1, 0])
 
 
-# def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoader, axes3d: plt.Axes) -> None:
-#     visualization_utils.draw_point_cloud_trajectory(axes3d, "title!", data_loader, 0)
+def test_draw_point_cloud_trajectory_no_error(data_loader: ArgoverseTrackingLoader, axes3d: plt.Axes) -> None:
+    visualization_utils.draw_point_cloud_trajectory(axes3d, "title!", data_loader, 0)
 
 
 def test_make_grid_ring_camera_no_error(data_loader: ArgoverseTrackingLoader) -> None: