From dc7adc2d11f4b752cc089e3ba7b57f0fbc3cce77 Mon Sep 17 00:00:00 2001 From: ly015 Date: Tue, 13 Apr 2021 11:16:43 +0800 Subject: [PATCH] Update H36M 3d keypoint evaluation metrics (#566) * Update H36M 3d keypoint evaluation * Add n-mpjpe metric * Remove align_root option * Refactor keypoint_mpjpe function * resolve comments --- .../h36m/linear_h36m_block2x1024_maxnorm.py | 3 +- mmpose/core/evaluation/eval_hooks.py | 2 +- mmpose/core/evaluation/pose3d_eval.py | 34 ++++++++--- .../datasets/body3d/body3d_h36m_dataset.py | 60 +++++++++++-------- tests/test_datasets/test_body3d_dataset.py | 4 +- 5 files changed, 67 insertions(+), 36 deletions(-) diff --git a/configs/body3d/simple_baseline/h36m/linear_h36m_block2x1024_maxnorm.py b/configs/body3d/simple_baseline/h36m/linear_h36m_block2x1024_maxnorm.py index 5c65f50427..57fe3d956a 100644 --- a/configs/body3d/simple_baseline/h36m/linear_h36m_block2x1024_maxnorm.py +++ b/configs/body3d/simple_baseline/h36m/linear_h36m_block2x1024_maxnorm.py @@ -4,7 +4,8 @@ dist_params = dict(backend='nccl') workflow = [('train', 1)] checkpoint_config = dict(interval=10) -evaluation = dict(interval=10, metric='mpjpe', key_indicator='mpjpe') +evaluation = dict( + interval=10, metric=['mpjpe', 'p-mpjpe'], key_indicator='mpjpe') # optimizer settings optimizer = dict( diff --git a/mmpose/core/evaluation/eval_hooks.py b/mmpose/core/evaluation/eval_hooks.py index ca2e3496cb..1a1a859c33 100644 --- a/mmpose/core/evaluation/eval_hooks.py +++ b/mmpose/core/evaluation/eval_hooks.py @@ -34,7 +34,7 @@ class EvalHook(Hook): rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y} init_value_map = {'greater': -inf, 'less': inf} greater_keys = ['acc', 'ap', 'ar', 'pck', 'auc'] - less_keys = ['loss', 'epe', 'nme', 'mpjpe', 'p-mpjpe'] + less_keys = ['loss', 'epe', 'nme', 'mpjpe', 'p-mpjpe', 'n-mpjpe'] def __init__(self, dataloader, diff --git a/mmpose/core/evaluation/pose3d_eval.py b/mmpose/core/evaluation/pose3d_eval.py index 1e75877239..34c112b94b 100644 --- a/mmpose/core/evaluation/pose3d_eval.py +++ b/mmpose/core/evaluation/pose3d_eval.py @@ -3,7 +3,7 @@ from .mesh_eval import compute_similarity_transform -def keypoint_mpjpe(pred, gt, mask): +def keypoint_mpjpe(pred, gt, mask, alignment='none'): """Calculate the mean per-joint position error (MPJPE) and the error after rigid alignment with the ground truth (P-MPJPE). @@ -17,6 +17,12 @@ def keypoint_mpjpe(pred, gt, mask): mask (np.ndarray[N, K]): Visibility of the target. False for invisible joints, and True for visible. Invisible joints will be ignored for accuracy calculation. + alignment (str, optional): method to align the prediction with the + groundtruth. Supported options are: + - ``'none'``: no alignment will be applied + - ``'scale'``: align in the least-square sense in scale + - ``'procrustes'``: align in the least-square sense in scale, + rotation and translation. Returns: tuple: A tuple containing joint position errors @@ -25,12 +31,22 @@ def keypoint_mpjpe(pred, gt, mask): ground truth """ assert mask.any() - pred_aligned = np.stack([ - compute_similarity_transform(pred_i, gt_i) - for pred_i, gt_i in zip(pred, gt) - ]) - mpjpe = np.linalg.norm(pred - gt, ord=2, axis=-1)[mask].mean() - p_mpjpe = np.linalg.norm(pred_aligned - gt, ord=2, axis=-1)[mask].mean() - - return mpjpe, p_mpjpe + if alignment == 'none': + pass + elif alignment == 'procrustes': + pred = np.stack([ + compute_similarity_transform(pred_i, gt_i) + for pred_i, gt_i in zip(pred, gt) + ]) + elif alignment == 'scale': + pred_dot_pred = np.einsum('nkc,nkc->n', pred, pred) + pred_dot_gt = np.einsum('nkc,nkc->n', pred, gt) + scale_factor = pred_dot_gt / pred_dot_pred + pred = pred * scale_factor[:, None, None] + else: + raise ValueError(f'Invalid value for alignment: {alignment}') + + error = np.linalg.norm(pred - gt, ord=2, axis=-1)[mask].mean() + + return error diff --git a/mmpose/datasets/datasets/body3d/body3d_h36m_dataset.py b/mmpose/datasets/datasets/body3d/body3d_h36m_dataset.py index 951cfbc6af..d5967eb46c 100644 --- a/mmpose/datasets/datasets/body3d/body3d_h36m_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_h36m_dataset.py @@ -61,7 +61,7 @@ class Body3DH36MDataset(Body3DBaseDataset): SUPPORTED_JOINT_2D_SRC = {'gt', 'detection', 'pipeline'} # metric - ALLOWED_METRICS = {'mpjpe', 'n-mpjpe'} + ALLOWED_METRICS = {'mpjpe', 'p-mpjpe', 'n-mpjpe'} def load_config(self, data_cfg): super().load_config(data_cfg) @@ -188,24 +188,30 @@ def evaluate(self, for _metric in metrics: if _metric == 'mpjpe': _nv_tuples = self._report_mpjpe(kpts) + elif _metric == 'p-mpjpe': + _nv_tuples = self._report_mpjpe(kpts, mode='p-mpjpe') elif _metric == 'n-mpjpe': - _nv_tuples = self._report_mpjpe(kpts, align_root=False) + _nv_tuples = self._report_mpjpe(kpts, mode='n-mpjpe') else: raise NotImplementedError name_value_tuples.extend(_nv_tuples) return OrderedDict(name_value_tuples) - def _report_mpjpe(self, keypoint_results, align_root=True): - """Keypoint evaluation. - - Report mean per joint position error (MPJPE) and mean per joint - position error after rigid alignment (MPJPE-PA) + def _report_mpjpe(self, keypoint_results, mode='mpjpe'): + """Cauculate mean per joint position error (MPJPE) or its variants like + P-MPJPE or N-MPJPE. Args: - align_root (bool): If true (default), the prediction will be - aligned to the ground truth by translation according to the - root joint. + keypoint_results (list): Keypoint predictions. See + 'Body3DH36MDataset.evaluate' for details. + mode (str): Specify mpjpe variants. Supported options are: + - ``'mpjpe'``: Standard MPJPE. + - ``'p-mpjpe'``: MPJPE after aligning prediction to groundtruth + via a rigid transformation (scale, rotation and + translation). + - ``'n-mpjpe'``: MPJPE after aligning prediction to groundtruth + in scale only. """ preds = [] @@ -230,23 +236,29 @@ def _report_mpjpe(self, keypoint_results, align_root=True): gts = np.stack(gts) masks = np.stack(masks).squeeze(-1) > 0 - if align_root: - # The prediction and groundtruth are aligned by subtracting root - # position respectively (equivalent to move the prediction to the - # groundtruth root position by translation). Then theroot joint - # will be removed from the pose. - preds = preds[:, 1:, :] - preds[:, :1, :] - gts = gts[:, 1:, :] - gts[:, :1, :] - masks = masks[:, 1:] + # Zero-center the pose around its root joint and remove root + # joint from the pose. + preds = preds[:, 1:, :] - preds[:, :1, :] + gts = gts[:, 1:, :] - gts[:, :1, :] + masks = masks[:, 1:] + + err_name = mode.upper() + if mode == 'mpjpe': + alignment = 'none' + elif mode == 'p-mpjpe': + alignment = 'procrustes' + elif mode == 'n-mpjpe': + alignment = 'scale' + else: + raise ValueError(f'Invalid mode: {mode}') - mpjpe, p_mpjpe = keypoint_mpjpe(preds, gts, masks) - name_value_tuples = [('MPJPE', mpjpe), ('P-MPJPE', p_mpjpe)] + error = keypoint_mpjpe(preds, gts, masks, alignment) + name_value_tuples = [(err_name, error)] for action_category, indices in action_category_indices.items(): - _mpjpe, _p_mpjpe = keypoint_mpjpe(preds[indices], gts[indices], - masks[indices]) - name_value_tuples.append((f'MPJPE_{action_category}', _mpjpe)) - name_value_tuples.append((f'P-MPJPE_{action_category}', _p_mpjpe)) + _error = keypoint_mpjpe(preds[indices], gts[indices], + masks[indices]) + name_value_tuples.append((f'{err_name}_{action_category}', _error)) return name_value_tuples diff --git a/tests/test_datasets/test_body3d_dataset.py b/tests/test_datasets/test_body3d_dataset.py index 3a2a24d161..6a218816ff 100644 --- a/tests/test_datasets/test_body3d_dataset.py +++ b/tests/test_datasets/test_body3d_dataset.py @@ -45,7 +45,9 @@ def test_body3d_h36m_dataset(): 'target_image_paths': [result['target_image_path']], }) - infos = custom_dataset.evaluate(outputs, tmpdir, 'mpjpe') + metrics = ['mpjpe', 'p-mpjpe', 'n-mpjpe'] + infos = custom_dataset.evaluate(outputs, tmpdir, metrics) np.testing.assert_almost_equal(infos['MPJPE'], 0.0) np.testing.assert_almost_equal(infos['P-MPJPE'], 0.0) + np.testing.assert_almost_equal(infos['N-MPJPE'], 0.0)