From acf63f2596b86ec5b1275c1702bf37ffc1ab7340 Mon Sep 17 00:00:00 2001 From: lupeng Date: Tue, 4 Apr 2023 09:58:02 +0800 Subject: [PATCH 1/3] fix CocoMetric and update yolo-pose results --- mmpose/evaluation/metrics/coco_metric.py | 12 +++++++++--- projects/yolox-pose/README.md | 8 ++++---- .../configs/yolox-pose_s_8xb32-300e_coco.py | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mmpose/evaluation/metrics/coco_metric.py b/mmpose/evaluation/metrics/coco_metric.py index 9b8bb706f2..dd059bf8fb 100644 --- a/mmpose/evaluation/metrics/coco_metric.py +++ b/mmpose/evaluation/metrics/coco_metric.py @@ -178,13 +178,19 @@ def process(self, data_batch: Sequence[dict], pred['keypoint_scores'] = keypoint_scores pred['category_id'] = data_sample.get('category_id', 1) - if ('bbox_scores' not in data_sample['gt_instances'] - or len(data_sample['gt_instances']['bbox_scores']) != - len(keypoints)): + if 'bbox_scores' in data_sample['pred_instances']: + # some one-stage models will predict bboxes and scores + # together with keypoints + bbox_scores = data_sample['pred_instances']['bbox_scores'] + elif ('bbox_scores' not in data_sample['gt_instances'] + or len(data_sample['gt_instances']['bbox_scores']) != + len(keypoints)): # bottom-up models might output different number of # instances from annotation bbox_scores = np.ones(len(keypoints)) else: + # top-down models use detected bboxes, the scores of which + # are contained in the gt_instances bbox_scores = data_sample['gt_instances']['bbox_scores'] pred['bbox_scores'] = bbox_scores diff --git a/projects/yolox-pose/README.md b/projects/yolox-pose/README.md index 828ea8af91..6b632c00ff 100644 --- a/projects/yolox-pose/README.md +++ b/projects/yolox-pose/README.md @@ -91,10 +91,10 @@ Results on COCO val2017 | Model | Input Size | AP | AP50 | AP75 | AR | AR50 | Download | | :-------------------------------------------------------------: | :--------: | :---: | :-------------: | :-------------: | :---: | :-------------: | :----------------------------------------------------------------------: | -| [YOLOX-tiny-Pose](./configs/yolox-pose_tiny_4xb64-300e_coco.py) | 640 | 0.477 | 0.756 | 0.506 | 0.547 | 0.802 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_tiny_4xb64-300e_coco-c47dd83b_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_tiny_4xb64-300e_coco_20230321.json) | -| [YOLOX-s-Pose](./configs/yolox-pose_s_8xb32-300e_coco.py) | 640 | 0.595 | 0.836 | 0.653 | 0.658 | 0.878 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_s_8xb32-300e_coco-9f5e3924_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_s_8xb32-300e_coco_20230321.json) | -| [YOLOX-m-Pose](./configs/yolox-pose_m_4xb64-300e_coco.py) | 640 | 0.659 | 0.870 | 0.729 | 0.713 | 0.903 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_m_4xb64-300e_coco-cbd11d30_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_m_4xb64-300e_coco_20230321.json) | -| [YOLOX-l-Pose](./configs/yolox-pose_l_4xb64-300e_coco.py) | 640 | 0.679 | 0.882 | 0.749 | 0.733 | 0.911 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_l_4xb64-300e_coco-122e4cf8_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_l_4xb64-300e_coco_20230321.json) | +| [YOLOX-tiny-Pose](./configs/yolox-pose_tiny_4xb64-300e_coco.py) | 640 | 0.518 | 0.799 | 0.545 | 0.566 | 0.841 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_tiny_4xb64-300e_coco-c47dd83b_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_tiny_4xb64-300e_coco_20230321.json) | +| [YOLOX-s-Pose](./configs/yolox-pose_s_8xb32-300e_coco.py) | 640 | 0.632 | 0.875 | 0.692 | 0.676 | 0.907 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_s_8xb32-300e_coco-9f5e3924_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_s_8xb32-300e_coco_20230321.json) | +| [YOLOX-m-Pose](./configs/yolox-pose_m_4xb64-300e_coco.py) | 640 | 0.685 | 0.897 | 0.753 | 0.727 | 0.925 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_m_4xb64-300e_coco-cbd11d30_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_m_4xb64-300e_coco_20230321.json) | +| [YOLOX-l-Pose](./configs/yolox-pose_l_4xb64-300e_coco.py) | 640 | 0.706 | 0.907 | 0.775 | 0.747 | 0.934 | [model](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_l_4xb64-300e_coco-122e4cf8_20230321.pth) \| [log](https://download.openmmlab.com/mmpose/v1/projects/yolox-pose/yolox-pose_l_4xb64-300e_coco_20230321.json) | We have only trained models with an input size of 640, as we couldn't replicate the performance enhancement mentioned in the paper when increasing the input size from 640 to 960. We warmly welcome any contributions if you can successfully reproduce the results from the paper! diff --git a/projects/yolox-pose/configs/yolox-pose_s_8xb32-300e_coco.py b/projects/yolox-pose/configs/yolox-pose_s_8xb32-300e_coco.py index d5ca77bc6c..f0cda72544 100644 --- a/projects/yolox-pose/configs/yolox-pose_s_8xb32-300e_coco.py +++ b/projects/yolox-pose/configs/yolox-pose_s_8xb32-300e_coco.py @@ -83,7 +83,7 @@ test_cfg=dict( yolox_style=True, multi_label=False, - score_thr=0.2, + score_thr=0.001, max_per_img=300, nms=dict(type='nms', iou_threshold=0.65))) @@ -189,7 +189,7 @@ val_evaluator = dict( type='mmpose.CocoMetric', ann_file=data_root + 'annotations/person_keypoints_val2017.json', -) + score_mode='bbox') test_evaluator = val_evaluator default_hooks = dict(checkpoint=dict(save_best='coco/AP', rule='greater')) From 295308a05938388f0a8fdc3c768c6313f1a945c3 Mon Sep 17 00:00:00 2001 From: lupeng Date: Tue, 4 Apr 2023 12:47:00 +0800 Subject: [PATCH 2/3] refine code of visualizer --- demo/docs/2d_human_pose_demo.md | 4 ++-- mmpose/visualization/local_visualizer.py | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/demo/docs/2d_human_pose_demo.md b/demo/docs/2d_human_pose_demo.md index 963ba25a92..f23d6d5b85 100644 --- a/demo/docs/2d_human_pose_demo.md +++ b/demo/docs/2d_human_pose_demo.md @@ -93,7 +93,7 @@ python demo/topdown_demo_with_mmdet.py \ demo/mmdetection_cfg/faster_rcnn_r50_fpn_coco.py \ https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth \ configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w32_8xb64-210e_coco-256x192.py \ - https://download.openmmlab.com/mmpose/top_down/hrnet/hrnet_w32_coco_256x192-c78dce93_20200708.pth \ + https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w32_8xb64-210e_coco-256x192-81c58e40_20220909.pth \ --input tests/data/posetrack18/videos/000001_mpiinew_test/000001_mpiinew_test.mp4 \ --output-root=vis_results/demo --show --draw-heatmap ``` @@ -116,7 +116,7 @@ Example: ```shell python demo/bottomup_demo.py \ configs/body_2d_keypoint/dekr/coco/dekr_hrnet-w32_8xb10-140e_coco-512x512.py \ - https://download.openmmlab.com/mmpose/bottom_up/dekr/hrnet_w32_coco_512x512-2a3056de_20220928.pth \ + https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/dekr/coco/dekr_hrnet-w32_8xb10-140e_coco-512x512_ac7c17bf-20221228.pth \ --input tests/data/coco/000000197388.jpg --output-root=vis_results \ --show --save-predictions ``` diff --git a/mmpose/visualization/local_visualizer.py b/mmpose/visualization/local_visualizer.py index 5a240f2174..0d6fcc61f4 100644 --- a/mmpose/visualization/local_visualizer.py +++ b/mmpose/visualization/local_visualizer.py @@ -264,12 +264,9 @@ def _draw_instances_kpts(self, # compute neck joint neck = np.mean(keypoints_info[:, [5, 6]], axis=1) # neck score when visualizing pred - neck[:, 2] = 1 - np.logical_or( - keypoints_info[:, 5, 2] < kpt_thr, - keypoints_info[:, 6, 2] < kpt_thr).astype(int) - neck[:, 3] = 1 - np.logical_or( - keypoints_info[:, 5, 3] < kpt_thr, - keypoints_info[:, 6, 3] < kpt_thr).astype(int) + neck[:, 2:4] = np.logical_and( + keypoints_info[:, 5, 2:4] < kpt_thr, + keypoints_info[:, 6, 2:4] < kpt_thr).astype(int) new_keypoints_info = np.insert( keypoints_info, 17, neck, axis=1) From c09fc5f199bbff4a67e833d730cb35ca64e1c4f9 Mon Sep 17 00:00:00 2001 From: lupeng Date: Tue, 4 Apr 2023 14:33:20 +0800 Subject: [PATCH 3/3] inferencer support filtering predicted instance via bbox_thr --- mmpose/apis/inferencers/mmpose_inferencer.py | 2 ++ mmpose/apis/inferencers/pose2d_inferencer.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mmpose/apis/inferencers/mmpose_inferencer.py b/mmpose/apis/inferencers/mmpose_inferencer.py index f5b23fb125..5727474a2b 100644 --- a/mmpose/apis/inferencers/mmpose_inferencer.py +++ b/mmpose/apis/inferencers/mmpose_inferencer.py @@ -176,6 +176,8 @@ def __call__( inputs = self.preprocess( inputs, batch_size=batch_size, **preprocess_kwargs) + forward_kwargs['bbox_thr'] = preprocess_kwargs.get('bbox_thr', -1) + preds = [] if 'pose2d' not in self.mode or not hasattr(self.pose2d_inferencer, 'detector'): diff --git a/mmpose/apis/inferencers/pose2d_inferencer.py b/mmpose/apis/inferencers/pose2d_inferencer.py index c55ad0043a..4f6771a49e 100644 --- a/mmpose/apis/inferencers/pose2d_inferencer.py +++ b/mmpose/apis/inferencers/pose2d_inferencer.py @@ -184,10 +184,15 @@ def preprocess_single(self, return data_infos - def forward(self, inputs: Union[dict, tuple]): + def forward(self, inputs: Union[dict, tuple], bbox_thr=-1): data_samples = super().forward(inputs) if self.cfg.data_mode == 'topdown': data_samples = [merge_data_samples(data_samples)] + if bbox_thr > 0: + for ds in data_samples: + if 'bbox_scores' in ds.pred_instances: + ds.pred_instances = ds.pred_instances[ + ds.pred_instances.bbox_scores > bbox_thr] return data_samples def __call__( @@ -241,6 +246,7 @@ def __call__( else: inputs = self._inputs_to_list(inputs) + forward_kwargs['bbox_thr'] = preprocess_kwargs.get('bbox_thr', -1) inputs = self.preprocess( inputs, batch_size=batch_size, **preprocess_kwargs)