[Fix] Include images without instance into bottom-up datasets for val…

…idation (open-mmlab#1936)

mmpose/datasets/datasets/base/base_coco_style_dataset.py

@@ -3,7 +3,7 @@
 import os.path as osp
 from copy import deepcopy
 from itertools import filterfalse, groupby
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 from mmengine.dataset import BaseDataset, force_full_init
@@ -169,16 +169,17 @@ def load_data_list(self) -> List[dict]:
         if self.bbox_file:
             data_list = self._load_detection_results()
         else:
-            data_list = self._load_annotations()
+            instance_list, image_list = self._load_annotations()
 
             if self.data_mode == 'topdown':
-                data_list = self._get_topdown_data_infos(data_list)
+                data_list = self._get_topdown_data_infos(instance_list)
             else:
-                data_list = self._get_bottomup_data_infos(data_list)
+                data_list = self._get_bottomup_data_infos(
+                    instance_list, image_list)
 
         return data_list
 
-    def _load_annotations(self):
+    def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in COCO format."""
 
         check_file_exist(self.ann_file)
@@ -188,22 +189,31 @@ def _load_annotations(self):
         # and each dict contains the 'id', 'name', etc. about this category
         self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds())
 
-        data_list = []
+        instance_list = []
+        image_list = []
 
         for img_id in coco.getImgIds():
             img = coco.loadImgs(img_id)[0]
+            img.update({
+                'img_id':
+                img_id,
+                'img_path':
+                osp.join(self.data_prefix['img'], img['file_name']),
+            })
+            image_list.append(img)
+
             ann_ids = coco.getAnnIds(imgIds=img_id)
             for ann in coco.loadAnns(ann_ids):
 
-                data_info = self.parse_data_info(
+                instance_info = self.parse_data_info(
                     dict(raw_ann_info=ann, raw_img_info=img))
 
                 # skip invalid instance annotation.
-                if not data_info:
+                if not instance_info:
                     continue
 
-                data_list.append(data_info)
-        return data_list
+                instance_list.append(instance_info)
+        return instance_list, image_list
 
     def parse_data_info(self, raw_data_info: dict) -> Optional[dict]:
         """Parse raw COCO annotation of an instance.
@@ -227,7 +237,6 @@ def parse_data_info(self, raw_data_info: dict) -> Optional[dict]:
         if 'bbox' not in ann or 'keypoints' not in ann:
             return None
 
-        img_path = osp.join(self.data_prefix['img'], img['file_name'])
         img_w, img_h = img['width'], img['height']
 
         # get bbox in shape [1, 4], formatted as xywh
@@ -252,7 +261,7 @@ def parse_data_info(self, raw_data_info: dict) -> Optional[dict]:
 
         data_info = {
             'img_id': ann['image_id'],
-            'img_path': img_path,
+            'img_path': img['img_path'],
             'bbox': bbox,
             'bbox_score': np.ones(1, dtype=np.float32),
             'num_keypoints': num_keypoints,
@@ -294,21 +303,26 @@ def _is_valid_instance(data_info: Dict) -> bool:
                 return False
         return True
 
-    def _get_topdown_data_infos(self, data_list: List[Dict]) -> List[Dict]:
+    def _get_topdown_data_infos(self, instance_list: List[Dict]) -> List[Dict]:
         """Organize the data list in top-down mode."""
         # sanitize data samples
-        data_list_tp = list(filter(self._is_valid_instance, data_list))
+        data_list_tp = list(filter(self._is_valid_instance, instance_list))
 
         return data_list_tp
 
-    def _get_bottomup_data_infos(self, data_list):
+    def _get_bottomup_data_infos(self, instance_list: List[Dict],
+                                 image_list: List[Dict]) -> List[Dict]:
         """Organize the data list in bottom-up mode."""
 
         # bottom-up data list
         data_list_bu = []
 
+        used_img_ids = set()
+
         # group instances by img_id
-        for img_id, data_infos in groupby(data_list, lambda x: x['img_id']):
+        for img_id, data_infos in groupby(instance_list,
+                                          lambda x: x['img_id']):
+            used_img_ids.add(img_id)
             data_infos = list(data_infos)
 
             # image data
@@ -336,6 +350,18 @@ def _get_bottomup_data_infos(self, data_list):
 
             data_list_bu.append(data_info_bu)
 
+        # add images without instance for evaluation
+        if self.test_mode:
+            for img_info in image_list:
+                if img_info['img_id'] not in used_img_ids:
+                    data_info_bu = {
+                        'img_id': img_info['img_id'],
+                        'img_path': img_info['img_path'],
+                        'id': list(),
+                        'raw_ann_info': None,
+                    }
+                    data_list_bu.append(data_info_bu)
+
         return data_list_bu
 
     def _load_detection_results(self) -> List[dict]:

mmpose/datasets/datasets/body/mpii_dataset.py

@@ -1,7 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import json
 import os.path as osp
-from typing import Callable, List, Optional, Sequence, Union
+from typing import Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 from mmengine.utils import check_file_exist
@@ -134,7 +134,7 @@ def __init__(self,
             lazy_init=lazy_init,
             max_refetch=max_refetch)
 
-    def _load_annotations(self) -> List[dict]:
+    def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in MPII format."""
 
         check_file_exist(self.ann_file)
@@ -148,7 +148,9 @@ def _load_annotations(self) -> List[dict]:
                                          [2, 0, 1])
             SC_BIAS = 0.6
 
-        data_list = []
+        instance_list = []
+        image_list = []
+        used_img_ids = set()
         ann_id = 0
 
         # mpii bbox scales are normalized with factor 200.
@@ -176,7 +178,7 @@ def _load_annotations(self) -> List[dict]:
             keypoints = np.array(ann['joints']).reshape(1, -1, 2)
             keypoints_visible = np.array(ann['joints_vis']).reshape(1, -1)
 
-            data_info = {
+            instance_info = {
                 'id': ann_id,
                 'img_id': int(ann['image'].split('.')[0]),
                 'img_path': osp.join(self.data_prefix['img'], ann['image']),
@@ -193,9 +195,16 @@ def _load_annotations(self) -> List[dict]:
                 headbox = headboxes_src[idx]
                 head_size = np.linalg.norm(headbox[1] - headbox[0], axis=0)
                 head_size *= SC_BIAS
-                data_info['head_size'] = head_size.reshape(1, -1)
+                instance_info['head_size'] = head_size.reshape(1, -1)
 
-            data_list.append(data_info)
+            if instance_info['img_id'] not in used_img_ids:
+                used_img_ids.add(instance_info['img_id'])
+                image_list.append({
+                    'img_id': instance_info['img_id'],
+                    'img_path': instance_info['img_path'],
+                })
+
+            instance_list.append(instance_info)
             ann_id = ann_id + 1
 
-        return data_list
+        return instance_list, image_list

mmpose/datasets/datasets/body/mpii_trb_dataset.py

@@ -1,7 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import json
 import os.path as osp
-from typing import List
+from typing import List, Tuple
 
 import numpy as np
 from mmengine.utils import check_file_exist
@@ -103,7 +103,7 @@ class MpiiTrbDataset(BaseCocoStyleDataset):
 
     METAINFO: dict = dict(from_file='configs/_base_/datasets/mpii_trb.py')
 
-    def _load_annotations(self) -> List[dict]:
+    def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in MPII-TRB format."""
 
         check_file_exist(self.ann_file)
@@ -112,7 +112,9 @@ def _load_annotations(self) -> List[dict]:
 
         imgid2info = {img['id']: img for img in data['images']}
 
-        data_list = []
+        instance_list = []
+        image_list = []
+        used_img_ids = set()
 
         # mpii-trb bbox scales are normalized with factor 200.
         pixel_std = 200.
@@ -135,7 +137,7 @@ def _load_annotations(self) -> List[dict]:
             img_path = osp.join(self.data_prefix['img'],
                                 imgid2info[img_id]['file_name'])
 
-            data_info = {
+            instance_info = {
                 'id': ann['id'],
                 'img_id': img_id,
                 'img_path': img_path,
@@ -151,10 +153,16 @@ def _load_annotations(self) -> List[dict]:
 
             # val set
             if 'headbox' in ann:
-                data_info['headbox'] = np.array(
+                instance_info['headbox'] = np.array(
                     ann['headbox'], dtype=np.float32)
 
-            data_list.append(data_info)
+            instance_list.append(instance_info)
+            if instance_info['img_id'] not in used_img_ids:
+                used_img_ids.add(instance_info['img_id'])
+                image_list.append({
+                    'img_id': instance_info['img_id'],
+                    'img_path': instance_info['img_path'],
+                })
 
-        data_list = sorted(data_list, key=lambda x: x['id'])
-        return data_list
+        instance_list = sorted(instance_list, key=lambda x: x['id'])
+        return instance_list, image_list

mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import os.path as osp
-from typing import List
+from typing import List, Tuple
 
 import numpy as np
 from mmengine.utils import check_file_exist
@@ -84,17 +84,27 @@ class CocoWholeBodyHandDataset(BaseCocoStyleDataset):
     METAINFO: dict = dict(
         from_file='configs/_base_/datasets/coco_wholebody_hand.py')
 
-    def _load_annotations(self) -> List[dict]:
+    def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in COCO format."""
 
         check_file_exist(self.ann_file)
 
         coco = COCO(self.ann_file)
-        data_list = []
+        instance_list = []
+        image_list = []
         id = 0
 
         for img_id in coco.getImgIds():
             img = coco.loadImgs(img_id)[0]
+
+            img.update({
+                'img_id':
+                img_id,
+                'img_path':
+                osp.join(self.data_prefix['img'], img['file_name']),
+            })
+            image_list.append(img)
+
             ann_ids = coco.getAnnIds(imgIds=img_id, iscrowd=False)
             anns = coco.loadAnns(ann_ids)
             for ann in anns:
@@ -103,8 +113,6 @@ def _load_annotations(self) -> List[dict]:
                     # valid instances (left and right hand) in one image
                     if ann[f'{type}hand_valid'] and max(
                             ann[f'{type}hand_kpts']) > 0:
-                        img_path = osp.join(self.data_prefix['img'],
-                                            img['file_name'])
 
                         bbox_xywh = np.array(
                             ann[f'{type}hand_box'],
@@ -120,9 +128,9 @@ def _load_annotations(self) -> List[dict]:
 
                         num_keypoints = np.count_nonzero(keypoints.max(axis=2))
 
-                        data_info = {
+                        instance_info = {
                             'img_id': ann['image_id'],
-                            'img_path': img_path,
+                            'img_path': img['img_path'],
                             'bbox': bbox,
                             'bbox_score': np.ones(1, dtype=np.float32),
                             'num_keypoints': num_keypoints,
@@ -132,8 +140,8 @@ def _load_annotations(self) -> List[dict]:
                             'segmentation': ann['segmentation'],
                             'id': id,
                         }
-                        data_list.append(data_info)
+                        instance_list.append(instance_info)
                         id = id + 1
 
-        data_list = sorted(data_list, key=lambda x: x['id'])
-        return data_list
+        instance_list = sorted(instance_list, key=lambda x: x['id'])
+        return instance_list, image_list

mmpose/evaluation/metrics/coco_metric.py

@@ -177,11 +177,15 @@ def process(self, data_batch: Sequence[dict],
             pred['keypoints'] = keypoints
             pred['keypoint_scores'] = keypoint_scores
             pred['category_id'] = data_sample.get('category_id', 1)
-            bbox_scores = data_sample['gt_instances']['bbox_scores']
-            if len(bbox_scores) != len(keypoints):
+
+            if ('bbox_scores' not in data_sample['gt_instances']
+                    or len(data_sample['gt_instances']['bbox_scores']) !=
+                    len(keypoints)):
                 # bottom-up models might output different number of
                 # instances from annotation
                 bbox_scores = np.ones(len(keypoints))
+            else:
+                bbox_scores = data_sample['gt_instances']['bbox_scores']
             pred['bbox_scores'] = bbox_scores
 
             # get area information
@@ -269,6 +273,11 @@ def gt_to_coco_json(self, gt_dicts: Sequence[dict],
 
             # filter duplicate annotations
             for ann in gt_dict['raw_ann_info']:
+                if ann is None:
+                    # during evaluation on bottom-up datasets, some images
+                    # do not have instance annotation
+                    continue
+
                 annotation = dict(
                     id=ann['id'],
                     image_id=ann['image_id'],

tests/test_datasets/test_datasets/test_dataset_wrappers/test_combined_dataset.py

@@ -86,6 +86,4 @@ def test_prepare_data(self):
         lens = dataset._lens
 
         data_info = dataset[lens[0]]
-        print(data_info.keys())
-        print(data_info['dataset_keypoint_weights'])
         self.check_data_info_keys(data_info)

tests/test_datasets/test_datasets/test_face_datasets/test_coco_wholebody_face_dataset.py

@@ -107,15 +107,15 @@ def test_topdown(self):
     def test_bottomup(self):
         # test bottomup training
         dataset = self.build_coco_wholebody_face_dataset(data_mode='bottomup')
-        # filter one invalid insances due to face_valid = false
+        # filter one invalid instance due to face_valid = false
         self.assertEqual(len(dataset), 3)
         self.check_data_info_keys(dataset[0], data_mode='bottomup')
 
         # test bottomup testing
         dataset = self.build_coco_wholebody_face_dataset(
             data_mode='bottomup', test_mode=True)
-        # filter invalid insances due to face_valid = false
-        self.assertEqual(len(dataset), 3)
+        # all images are used for evaluation
+        self.assertEqual(len(dataset), 4)
         self.check_data_info_keys(dataset[0], data_mode='bottomup')
 
     def test_exceptions_and_warnings(self):