From 220520155ccc8d19451d7a0f20aad0908de3c91b Mon Sep 17 00:00:00 2001 From: Yifan Lareina WU Date: Fri, 7 Apr 2023 14:41:36 +0800 Subject: [PATCH] fileio from petrel (#2182) --- .../datasets/base/base_coco_style_dataset.py | 31 ++++++++++--------- mmpose/datasets/datasets/body/mpii_dataset.py | 18 ++++++----- .../datasets/body/mpii_trb_dataset.py | 13 ++++---- .../body/posetrack18_video_dataset.py | 16 +++++----- .../hand/coco_wholebody_hand_dataset.py | 15 ++++----- mmpose/evaluation/metrics/coco_metric.py | 5 +-- 6 files changed, 52 insertions(+), 46 deletions(-) diff --git a/mmpose/datasets/datasets/base/base_coco_style_dataset.py b/mmpose/datasets/datasets/base/base_coco_style_dataset.py index a1c3bc2f5c..3b592813d8 100644 --- a/mmpose/datasets/datasets/base/base_coco_style_dataset.py +++ b/mmpose/datasets/datasets/base/base_coco_style_dataset.py @@ -7,8 +7,8 @@ import numpy as np from mmengine.dataset import BaseDataset, force_full_init -from mmengine.fileio import load -from mmengine.utils import check_file_exist, is_list_of +from mmengine.fileio import exists, get_local_path, load +from mmengine.utils import is_list_of from xtcocotools.coco import COCO from mmpose.registry import DATASETS @@ -195,18 +195,19 @@ def load_data_list(self) -> List[dict]: def _load_annotations(self) -> Tuple[List[dict], List[dict]]: """Load data from annotations in COCO format.""" - check_file_exist(self.ann_file) + assert exists(self.ann_file), 'Annotation file does not exist' - coco = COCO(self.ann_file) + with get_local_path(self.ann_file) as local_path: + self.coco = COCO(local_path) # set the metainfo about categories, which is a list of dict # and each dict contains the 'id', 'name', etc. about this category - self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds()) + self._metainfo['CLASSES'] = self.coco.loadCats(self.coco.getCatIds()) instance_list = [] image_list = [] - for img_id in coco.getImgIds(): - img = coco.loadImgs(img_id)[0] + for img_id in self.coco.getImgIds(): + img = self.coco.loadImgs(img_id)[0] img.update({ 'img_id': img_id, @@ -215,8 +216,8 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]: }) image_list.append(img) - ann_ids = coco.getAnnIds(imgIds=img_id) - for ann in coco.loadAnns(ann_ids): + ann_ids = self.coco.getAnnIds(imgIds=img_id) + for ann in self.coco.loadAnns(ann_ids): instance_info = self.parse_data_info( dict(raw_ann_info=ann, raw_img_info=img)) @@ -380,18 +381,18 @@ def _get_bottomup_data_infos(self, instance_list: List[Dict], def _load_detection_results(self) -> List[dict]: """Load data from detection results with dummy keypoint annotations.""" - check_file_exist(self.ann_file) - check_file_exist(self.bbox_file) - + assert exists(self.ann_file), 'Annotation file does not exist' + assert exists(self.bbox_file), 'Bbox file does not exist' # load detection results det_results = load(self.bbox_file) assert is_list_of(det_results, dict) # load coco annotations to build image id-to-name index - coco = COCO(self.ann_file) + with get_local_path(self.ann_file) as local_path: + self.coco = COCO(local_path) # set the metainfo about categories, which is a list of dict # and each dict contains the 'id', 'name', etc. about this category - self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds()) + self._metainfo['CLASSES'] = self.coco.loadCats(self.coco.getCatIds()) num_keypoints = self.metainfo['num_keypoints'] data_list = [] @@ -401,7 +402,7 @@ def _load_detection_results(self) -> List[dict]: if det['category_id'] != 1: continue - img = coco.loadImgs(det['image_id'])[0] + img = self.coco.loadImgs(det['image_id'])[0] img_path = osp.join(self.data_prefix['img'], img['file_name']) bbox_xywh = np.array( diff --git a/mmpose/datasets/datasets/body/mpii_dataset.py b/mmpose/datasets/datasets/body/mpii_dataset.py index a1f777a10b..237f1ab2b6 100644 --- a/mmpose/datasets/datasets/body/mpii_dataset.py +++ b/mmpose/datasets/datasets/body/mpii_dataset.py @@ -4,7 +4,7 @@ from typing import Callable, List, Optional, Sequence, Tuple, Union import numpy as np -from mmengine.utils import check_file_exist +from mmengine.fileio import exists, get_local_path from scipy.io import loadmat from mmpose.registry import DATASETS @@ -137,14 +137,16 @@ def __init__(self, def _load_annotations(self) -> Tuple[List[dict], List[dict]]: """Load data from annotations in MPII format.""" - check_file_exist(self.ann_file) - with open(self.ann_file) as anno_file: - anns = json.load(anno_file) + assert exists(self.ann_file), 'Annotation file does not exist' + with get_local_path(self.ann_file) as local_path: + with open(local_path) as anno_file: + self.anns = json.load(anno_file) if self.headbox_file: - check_file_exist(self.headbox_file) - headbox_dict = loadmat(self.headbox_file) - headboxes_src = np.transpose(headbox_dict['headboxes_src'], + assert exists(self.headbox_file), 'Headbox file does not exist' + with get_local_path(self.headbox_file) as local_path: + self.headbox_dict = loadmat(local_path) + headboxes_src = np.transpose(self.headbox_dict['headboxes_src'], [2, 0, 1]) SC_BIAS = 0.6 @@ -156,7 +158,7 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]: # mpii bbox scales are normalized with factor 200. pixel_std = 200. - for idx, ann in enumerate(anns): + for idx, ann in enumerate(self.anns): center = np.array(ann['center'], dtype=np.float32) scale = np.array([ann['scale'], ann['scale']], dtype=np.float32) * pixel_std diff --git a/mmpose/datasets/datasets/body/mpii_trb_dataset.py b/mmpose/datasets/datasets/body/mpii_trb_dataset.py index be9154ad70..bb96ad876f 100644 --- a/mmpose/datasets/datasets/body/mpii_trb_dataset.py +++ b/mmpose/datasets/datasets/body/mpii_trb_dataset.py @@ -4,7 +4,7 @@ from typing import List, Tuple import numpy as np -from mmengine.utils import check_file_exist +from mmengine.fileio import exists, get_local_path from mmpose.registry import DATASETS from mmpose.structures.bbox import bbox_cs2xyxy @@ -106,11 +106,12 @@ class MpiiTrbDataset(BaseCocoStyleDataset): def _load_annotations(self) -> Tuple[List[dict], List[dict]]: """Load data from annotations in MPII-TRB format.""" - check_file_exist(self.ann_file) - with open(self.ann_file) as anno_file: - data = json.load(anno_file) + assert exists(self.ann_file), 'Annotation file does not exist' + with get_local_path(self.ann_file) as local_path: + with open(local_path) as anno_file: + self.data = json.load(anno_file) - imgid2info = {img['id']: img for img in data['images']} + imgid2info = {img['id']: img for img in self.data['images']} instance_list = [] image_list = [] @@ -119,7 +120,7 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]: # mpii-trb bbox scales are normalized with factor 200. pixel_std = 200. - for ann in data['annotations']: + for ann in self.data['annotations']: img_id = ann['image_id'] # center, scale in shape [1, 2] and bbox in [1, 4] diff --git a/mmpose/datasets/datasets/body/posetrack18_video_dataset.py b/mmpose/datasets/datasets/body/posetrack18_video_dataset.py index fc0b47e517..cc5fe8646c 100644 --- a/mmpose/datasets/datasets/body/posetrack18_video_dataset.py +++ b/mmpose/datasets/datasets/body/posetrack18_video_dataset.py @@ -3,8 +3,8 @@ from typing import Callable, List, Optional, Sequence, Union import numpy as np -from mmengine.fileio import load -from mmengine.utils import check_file_exist, is_list_of +from mmengine.fileio import exists, get_local_path, load +from mmengine.utils import is_list_of from xtcocotools.coco import COCO from mmpose.registry import DATASETS @@ -287,22 +287,22 @@ def parse_data_info(self, raw_data_info: dict) -> Optional[dict]: def _load_detection_results(self) -> List[dict]: """Load data from detection results with dummy keypoint annotations.""" - - check_file_exist(self.ann_file) - check_file_exist(self.bbox_file) + assert exists(self.ann_file), 'Annotation file does not exist' + assert exists(self.bbox_file), 'Bbox file does not exist' # load detection results det_results = load(self.bbox_file) assert is_list_of(det_results, dict) # load coco annotations to build image id-to-name index - coco = COCO(self.ann_file) + with get_local_path(self.ann_file) as local_path: + self.coco = COCO(local_path) # mapping image name to id name2id = {} # mapping image id to name id2name = {} - for img_id, image in coco.imgs.items(): + for img_id, image in self.coco.imgs.items(): file_name = image['file_name'] id2name[img_id] = file_name name2id[file_name] = img_id @@ -333,7 +333,7 @@ def _load_detection_results(self) -> List[dict]: img_id = name2id[det['image_name']] else: img_id = det['image_id'] - img_ann = coco.loadImgs(img_id)[0] + img_ann = self.coco.loadImgs(img_id)[0] nframes = int(img_ann['nframes']) # deal with multiple image paths diff --git a/mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py b/mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py index 1831c9c89d..dba0132f58 100644 --- a/mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py +++ b/mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py @@ -3,7 +3,7 @@ from typing import List, Tuple import numpy as np -from mmengine.utils import check_file_exist +from mmengine.fileio import exists, get_local_path from xtcocotools.coco import COCO from mmpose.registry import DATASETS @@ -87,15 +87,16 @@ class CocoWholeBodyHandDataset(BaseCocoStyleDataset): def _load_annotations(self) -> Tuple[List[dict], List[dict]]: """Load data from annotations in COCO format.""" - check_file_exist(self.ann_file) + assert exists(self.ann_file), 'Annotation file does not exist' - coco = COCO(self.ann_file) + with get_local_path(self.ann_file) as local_path: + self.coco = COCO(local_path) instance_list = [] image_list = [] id = 0 - for img_id in coco.getImgIds(): - img = coco.loadImgs(img_id)[0] + for img_id in self.coco.getImgIds(): + img = self.coco.loadImgs(img_id)[0] img.update({ 'img_id': @@ -105,8 +106,8 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]: }) image_list.append(img) - ann_ids = coco.getAnnIds(imgIds=img_id, iscrowd=False) - anns = coco.loadAnns(ann_ids) + ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=False) + anns = self.coco.loadAnns(ann_ids) for ann in anns: for type in ['left', 'right']: # filter invalid hand annotations, there might be two diff --git a/mmpose/evaluation/metrics/coco_metric.py b/mmpose/evaluation/metrics/coco_metric.py index dd059bf8fb..8327e2eca7 100644 --- a/mmpose/evaluation/metrics/coco_metric.py +++ b/mmpose/evaluation/metrics/coco_metric.py @@ -7,7 +7,7 @@ import numpy as np from mmengine.evaluator import BaseMetric -from mmengine.fileio import dump, load +from mmengine.fileio import dump, get_local_path, load from mmengine.logging import MMLogger from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval @@ -102,7 +102,8 @@ def __init__(self, # initialize coco helper with the annotation json file # if ann_file is not specified, initialize with the converted dataset if ann_file is not None: - self.coco = COCO(ann_file) + with get_local_path(ann_file) as local_path: + self.coco = COCO(local_path) else: self.coco = None