diff --git a/datumaro/plugins/market1501_format.py b/datumaro/plugins/market1501_format.py index 8f4e26cc70b0..2370f28ac994 100644 --- a/datumaro/plugins/market1501_format.py +++ b/datumaro/plugins/market1501_format.py @@ -2,80 +2,96 @@ # # SPDX-License-Identifier: MIT +import os import os.path as osp import re from distutils.util import strtobool -from glob import glob +from itertools import chain from datumaro.components.converter import Converter from datumaro.components.extractor import (DatasetItem, Importer, SourceExtractor) +from datumaro.util.image import find_images class Market1501Path: QUERY_DIR = 'query' BBOX_DIR = 'bounding_box_' IMAGE_EXT = '.jpg' - PATTERN = re.compile(r'([-\d]+)_c(\d)') - IMAGE_NAMES = 'images_' + PATTERN = re.compile(r'^(-?\d+)_c(\d+)(?:s\d+_\d+_00(.*))?') + LIST_PREFIX = 'images_' + UNKNOWN_ID = -1 class Market1501Extractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isdir(path): raise NotADirectoryError( "Can't open folder with annotation files '%s'" % path) - subset = '' - for dirname in glob(osp.join(path, '*')): - if osp.basename(dirname).startswith(Market1501Path.BBOX_DIR): - subset = osp.basename(dirname).replace(Market1501Path.BBOX_DIR, '') - if osp.basename(dirname).startswith(Market1501Path.IMAGE_NAMES): - subset = osp.basename(dirname).replace(Market1501Path.IMAGE_NAMES, '') - subset = osp.splitext(subset)[0] - break + if not subset: + subset = '' + for p in os.listdir(path): + pf = osp.join(path, p) + + if p.startswith(Market1501Path.BBOX_DIR) and osp.isdir(pf): + subset = p.replace(Market1501Path.BBOX_DIR, '') + break + + if p.startswith(Market1501Path.LIST_PREFIX) and osp.isfile(pf): + subset = p.replace(Market1501Path.LIST_PREFIX, '') + subset = osp.splitext(subset)[0] + break super().__init__(subset=subset) self._path = path self._items = list(self._load_items(path).values()) - def _load_items(self, path): + def _load_items(self, rootdir): items = {} - paths = glob(osp.join(path, Market1501Path.QUERY_DIR, '*')) - paths += glob(osp.join(path, Market1501Path.BBOX_DIR + self._subset, '*')) - - anno_file = osp.join(path, - Market1501Path.IMAGE_NAMES + self._subset + '.txt') - if len(paths) == 0 and osp.isfile(anno_file): + paths = [] + anno_file = osp.join(rootdir, + Market1501Path.LIST_PREFIX + self._subset + '.txt') + if osp.isfile(anno_file): with open(anno_file, encoding='utf-8') as f: for line in f: - paths.append(line.strip()) + paths.append(osp.join(rootdir, line.strip())) + else: + paths = list(chain( + find_images(osp.join(rootdir, + Market1501Path.QUERY_DIR), + recursive=True), + find_images(osp.join(rootdir, + Market1501Path.BBOX_DIR + self._subset), + recursive=True), + )) for image_path in paths: - if osp.splitext(image_path)[-1] != Market1501Path.IMAGE_EXT: - continue - - item_id = osp.splitext(osp.basename(image_path))[0] - pid, camid = -1, -1 - search = Market1501Path.PATTERN.search(image_path) + item_id = osp.splitext(osp.normpath(image_path))[0] + if osp.isabs(image_path): + item_id = osp.relpath(item_id, rootdir) + subdir, item_id = item_id.split(os.sep, maxsplit=1) + + pid = Market1501Path.UNKNOWN_ID + camid = Market1501Path.UNKNOWN_ID + search = Market1501Path.PATTERN.search(osp.basename(item_id)) if search: - pid, camid = map(int, search.groups()) - if 19 < len(item_id): - item_id = item_id[19:] - items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) - - if pid == -1: - continue - - attributes = items[item_id].attributes - camid -= 1 + pid, camid = map(int, search.groups()[0:2]) + camid -= 1 # make ids 0-based + custom_name = search.groups()[2] + if custom_name: + item_id = osp.join(osp.dirname(item_id), custom_name) + + item = items.get(item_id) + if item is None: + item = DatasetItem(id=item_id, subset=self._subset, + image=image_path) + items[item_id] = item + + attributes = item.attributes + attributes['query'] = subdir == Market1501Path.QUERY_DIR attributes['person_id'] = pid attributes['camera_id'] = camid - if osp.basename(osp.dirname(image_path)) == Market1501Path.QUERY_DIR: - attributes['query'] = True - else: - attributes['query'] = False return items class Market1501Importer(Importer): @@ -86,20 +102,23 @@ def find_sources(cls, path): return [{ 'url': path, 'format': 'market1501' }] class Market1501Converter(Converter): - DEFAULT_IMAGE_EXT = '.jpg' + DEFAULT_IMAGE_EXT = Market1501Path.IMAGE_EXT def apply(self): for subset_name, subset in self._extractor.subsets().items(): annotation = '' + for item in subset: image_name = item.id if Market1501Path.PATTERN.search(image_name) == None: if 'person_id' in item.attributes and \ 'camera_id' in item.attributes: image_pattern = '{:04d}_c{}s1_000000_00{}' - pid = int(item.attributes.get('person_id')) - camid = int(item.attributes.get('camera_id')) + 1 - image_name = image_pattern.format(pid, camid, item.id) + pid = int(item.attributes['person_id']) + camid = int(item.attributes['camera_id']) + 1 + dirname, basename = osp.split(item.id) + image_name = osp.join(dirname, + image_pattern.format(pid, camid, basename)) dirname = Market1501Path.BBOX_DIR + subset_name if 'query' in item.attributes: @@ -108,15 +127,15 @@ def apply(self): query = strtobool(query) if query: dirname = Market1501Path.QUERY_DIR - image_path = osp.join(self._save_dir, dirname, - image_name + Market1501Path.IMAGE_EXT) - if item.has_image and self._save_images: - self._save_image(item, image_path) - else: - annotation += '%s\n' % image_path - - if 0 < len(annotation): - annotation_file = osp.join(self._save_dir, - Market1501Path.IMAGE_NAMES + subset_name + '.txt') - with open(annotation_file, 'w') as f: - f.write(annotation) + + image_path = self._make_image_filename(item, + name=image_name, subdir=dirname) + if self._save_images and item.has_image: + self._save_image(item, osp.join(self._save_dir, image_path)) + + annotation += '%s\n' % image_path + + annotation_file = osp.join(self._save_dir, + Market1501Path.LIST_PREFIX + subset_name + '.txt') + with open(annotation_file, 'w') as f: + f.write(annotation) diff --git a/datumaro/plugins/vgg_face2_format.py b/datumaro/plugins/vgg_face2_format.py index 3c57413c53da..33b41021602d 100644 --- a/datumaro/plugins/vgg_face2_format.py +++ b/datumaro/plugins/vgg_face2_format.py @@ -9,6 +9,7 @@ from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points, SourceExtractor) +from datumaro.util.image import find_images class VggFace2Path: @@ -20,15 +21,16 @@ class VggFace2Path: IMAGES_DIR_NO_LABEL = 'no_label' class VggFace2Extractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isfile(path): raise Exception("Can't read .csv annotation file '%s'" % path) self._path = path self._dataset_dir = osp.dirname(osp.dirname(path)) - subset = osp.splitext(osp.basename(path))[0] - if subset.startswith(VggFace2Path.LANDMARKS_FILE): - subset = subset.split('_')[2] + if not subset: + subset = osp.splitext(osp.basename(path))[0] + if subset.startswith(VggFace2Path.LANDMARKS_FILE): + subset = subset.split('_')[2] super().__init__(subset=subset) self._categories = self._load_categories() @@ -68,6 +70,13 @@ def _split_item_path(path): items = {} + image_dir = osp.join(self._dataset_dir, self._subset) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + with open(path, encoding='utf-8') as content: landmarks_table = list(csv.DictReader(content)) for row in landmarks_table: @@ -77,10 +86,8 @@ def _split_item_path(path): item_id, label = _split_item_path(item_id) if item_id not in items: - image_path = osp.join(self._dataset_dir, self._subset, - row['NAME_ID'] + VggFace2Path.IMAGE_EXT) items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) + image=images.get(row['NAME_ID'])) annotations = items[item_id].annotations if [a for a in annotations if a.type == AnnotationType.points]: @@ -105,10 +112,8 @@ def _split_item_path(path): item_id, label = _split_item_path(item_id) if item_id not in items: - image_path = osp.join(self._dataset_dir, self._subset, - row['NAME_ID'] + VggFace2Path.IMAGE_EXT) items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) + image=images.get(row['NAME_ID'])) annotations = items[item_id].annotations if [a for a in annotations if a.type == AnnotationType.bbox]: diff --git a/datumaro/plugins/voc_format/extractor.py b/datumaro/plugins/voc_format/extractor.py index 84e9065614a7..993b825350f9 100644 --- a/datumaro/plugins/voc_format/extractor.py +++ b/datumaro/plugins/voc_format/extractor.py @@ -12,8 +12,8 @@ from datumaro.components.extractor import (SourceExtractor, DatasetItem, AnnotationType, Label, Mask, Bbox, CompiledMask ) -from datumaro.util import dir_items -from datumaro.util.image import Image +from datumaro.util.os_util import dir_items +from datumaro.util.image import Image, find_images from datumaro.util.mask_tools import lazy_mask, invert_colormap from .format import ( @@ -82,13 +82,19 @@ def __init__(self, path): def __iter__(self): raw_anns = self._load_annotations() + + image_dir = osp.join(self._dataset_dir, VocPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + for item_id in self._items: log.debug("Reading item '%s'" % item_id) - image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, - item_id + VocPath.IMAGE_EXT) anns = self._parse_annotations(raw_anns, item_id) yield DatasetItem(id=item_id, subset=self._subset, - image=image, annotations=anns) + image=images.get(item_id), annotations=anns) def _load_annotations(self): annotations = defaultdict(list) @@ -251,13 +257,18 @@ def __init__(self, path): super().__init__(path, task=VocTask.segmentation) def __iter__(self): + image_dir = osp.join(self._dataset_dir, VocPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + for item_id in self._items: log.debug("Reading item '%s'" % item_id) - image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, - item_id + VocPath.IMAGE_EXT) anns = self._load_annotations(item_id) yield DatasetItem(id=item_id, subset=self._subset, - image=image, annotations=anns) + image=images.get(item_id), annotations=anns) @staticmethod def _lazy_extract_mask(mask, c): diff --git a/datumaro/plugins/widerface_format.py b/datumaro/plugins/widerface_format.py index efb5b75ab8c9..f5e0008f6075 100644 --- a/datumaro/plugins/widerface_format.py +++ b/datumaro/plugins/widerface_format.py @@ -23,15 +23,16 @@ class WiderFacePath: 'occluded', 'pose', 'invalid'] class WiderFaceExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isfile(path): raise Exception("Can't read annotation file '%s'" % path) self._path = path self._dataset_dir = osp.dirname(osp.dirname(path)) - subset = osp.splitext(osp.basename(path))[0] - if re.fullmatch(r'wider_face_\S+_bbx_gt', subset): - subset = subset.split('_')[2] + if not subset: + subset = osp.splitext(osp.basename(path))[0] + if re.fullmatch(r'wider_face_\S+_bbx_gt', subset): + subset = subset.split('_')[2] super().__init__(subset=subset) self._categories = self._load_categories() @@ -65,15 +66,18 @@ def _load_items(self, path): with open(path, 'r', encoding='utf-8') as f: lines = f.readlines() - image_ids = [image_id for image_id, line in enumerate(lines) - if WiderFacePath.IMAGE_EXT in line] + line_ids = [line_idx for line_idx, line in enumerate(lines) + if ('/' in line or '\\' in line) and '.' in line] \ + # a heuristic for paths + + for line_idx in line_ids: + image_path = lines[line_idx].strip() + item_id = osp.splitext(image_path)[0] - for image_id in image_ids: - image = lines[image_id] image_path = osp.join(self._dataset_dir, WiderFacePath.SUBSET_DIR + self._subset, - WiderFacePath.IMAGES_DIR, image[:-1]) - item_id = image[:-(len(WiderFacePath.IMAGE_EXT) + 1)] + WiderFacePath.IMAGES_DIR, image_path) + annotations = [] if '/' in item_id: label_name = item_id.split('/')[0] @@ -85,8 +89,15 @@ def _load_items(self, path): annotations.append(Label(label=label)) item_id = item_id[len(item_id.split('/')[0]) + 1:] - bbox_count = lines[image_id + 1] - bbox_lines = lines[image_id + 2 : image_id + int(bbox_count) + 2] + items[item_id] = DatasetItem(id=item_id, subset=self._subset, + image=image_path, annotations=annotations) + + try: + bbox_count = int(lines[line_idx + 1]) # can be the next image + except ValueError: + continue + + bbox_lines = lines[line_idx + 2 : line_idx + bbox_count + 2] for bbox in bbox_lines: bbox_list = bbox.split() if 4 <= len(bbox_list): @@ -111,8 +122,6 @@ def _load_items(self, path): attributes=attributes, label=label )) - items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path, annotations=annotations) return items class WiderFaceImporter(Importer): diff --git a/datumaro/plugins/yolo_format/extractor.py b/datumaro/plugins/yolo_format/extractor.py index b67eb9cb8e5a..33ab8eb7ff51 100644 --- a/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/plugins/yolo_format/extractor.py @@ -10,7 +10,7 @@ from datumaro.components.extractor import (SourceExtractor, Extractor, DatasetItem, AnnotationType, Bbox, LabelCategories, Importer ) -from datumaro.util import split_path +from datumaro.util.os_util import split_path from datumaro.util.image import Image from .format import YoloPath @@ -55,7 +55,7 @@ def __init__(self, config_path, image_info=None): with open(image_info, encoding='utf-8') as f: image_info = {} for line in f: - image_name, h, w = line.strip().split() + image_name, h, w = line.strip().rsplit(maxsplit=2) image_info[image_name] = (int(h), int(w)) self._image_info = image_info @@ -107,7 +107,7 @@ def __init__(self, config_path, image_info=None): @staticmethod def localize_path(path): path = osp.normpath(path).strip() - default_base = osp.join('data', '') + default_base = 'data' + osp.sep if path.startswith(default_base): # default path path = path[len(default_base) : ] return path @@ -117,9 +117,9 @@ def name_from_path(cls, path): path = cls.localize_path(path) parts = split_path(path) if 1 < len(parts) and not osp.isabs(path): - # NOTE: when path is like [data/]/ + # NOTE: when path is like [data/]_obj/ # drop everything but - # can be , so no just basename() + # can be , so not just basename() path = osp.join(*parts[1:]) return osp.splitext(path)[0] diff --git a/datumaro/util/__init__.py b/datumaro/util/__init__.py index b7e56890ae32..d323662ef747 100644 --- a/datumaro/util/__init__.py +++ b/datumaro/util/__init__.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: MIT import attr -import os -import os.path as osp from contextlib import ExitStack from functools import partial, wraps from itertools import islice @@ -14,32 +12,6 @@ def find(iterable, pred=lambda x: True, default=None): return next((x for x in iterable if pred(x)), default) -def dir_items(path, ext, truncate_ext=False): - items = [] - for f in os.listdir(path): - ext_pos = f.rfind(ext) - if ext_pos != -1: - if truncate_ext: - f = f[:ext_pos] - items.append(f) - return items - -def split_path(path): - path = osp.normpath(path) - parts = [] - - while True: - path, part = osp.split(path) - if part: - parts.append(part) - else: - if path: - parts.append(path) - break - parts.reverse() - - return parts - def cast(value, type_conv, default=None): if value is None: return default diff --git a/datumaro/util/os_util.py b/datumaro/util/os_util.py index 6aa5c2774e39..094329206a17 100644 --- a/datumaro/util/os_util.py +++ b/datumaro/util/os_util.py @@ -45,4 +45,30 @@ def walk(path, max_depth=None): if baselevel + max_depth <= curlevel: dirnames.clear() # topdown=True allows to modify the list - yield dirpath, dirnames, filenames \ No newline at end of file + yield dirpath, dirnames, filenames + +def dir_items(path, ext, truncate_ext=False): + items = [] + for f in os.listdir(path): + ext_pos = f.rfind(ext) + if ext_pos != -1: + if truncate_ext: + f = f[:ext_pos] + items.append(f) + return items + +def split_path(path): + path = osp.normpath(path) + parts = [] + + while True: + path, part = osp.split(path) + if part: + parts.append(part) + else: + if path: + parts.append(path) + break + parts.reverse() + + return parts diff --git a/datumaro/util/test_utils.py b/datumaro/util/test_utils.py index 16334b72dbb9..8c5cf05af24c 100644 --- a/datumaro/util/test_utils.py +++ b/datumaro/util/test_utils.py @@ -108,7 +108,7 @@ def compare_datasets(test, expected, actual, ignored_attrs=None, item_b = find(actual, lambda x: x.id == item_a.id and \ x.subset == item_a.subset) test.assertFalse(item_b is None, item_a.id) - test.assertEqual(item_a.attributes, item_b.attributes) + test.assertEqual(item_a.attributes, item_b.attributes, item_a.id) if (require_images and item_a.has_image and item_a.image.has_data) or \ item_a.has_image and item_a.image.has_data and \ item_b.has_image and item_b.image.has_data: diff --git a/tests/test_market1501_format.py b/tests/test_market1501_format.py index a53ef89d6f4d..69f299edc37d 100644 --- a/tests/test_market1501_format.py +++ b/tests/test_market1501_format.py @@ -6,6 +6,7 @@ from datumaro.components.extractor import DatasetItem from datumaro.plugins.market1501_format import (Market1501Converter, Market1501Importer) +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -78,7 +79,8 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): Market1501Converter.convert(source_dataset, test_dir, save_images=True) parsed_dataset = Dataset.import_from(test_dir, 'market1501') - compare_datasets(self, source_dataset, parsed_dataset) + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ @@ -106,6 +108,31 @@ def test_can_save_dataset_with_no_save_images(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image( + path='q/1.JPEG', data=np.zeros((4, 3, 3))), + attributes={ + 'camera_id': 1, + 'person_id': 1, + 'query': False + }), + DatasetItem(id='a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.zeros((3, 4, 3))), + attributes={ + 'camera_id': 1, + 'person_id': 2, + 'query': True + }), + ]) + + with TestDir() as test_dir: + Market1501Converter.convert(expected, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'market1501') + + compare_datasets(self, expected, parsed_dataset, + require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'market1501_dataset') class Market1501ImporterTest(TestCase): diff --git a/tests/test_vgg_face2_format.py b/tests/test_vgg_face2_format.py index da95614d9a0b..d6d232a9217b 100644 --- a/tests/test_vgg_face2_format.py +++ b/tests/test_vgg_face2_format.py @@ -7,6 +7,7 @@ Label, LabelCategories, Points) from datumaro.plugins.vgg_face2_format import (VggFace2Converter, VggFace2Importer) +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -85,7 +86,8 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): VggFace2Converter.convert(source_dataset, test_dir, save_images=True) parsed_dataset = Dataset.import_from(test_dir, 'vgg_face2') - compare_datasets(self, source_dataset, parsed_dataset) + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ @@ -147,6 +149,26 @@ def test_can_save_dataset_with_wrong_number_of_points(self): compare_datasets(self, target_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3))), + annotations=[ + Bbox(0, 2, 4, 2, label=0), + Points([4.23, 4.32, 5.34, 4.45, 3.54, + 3.56, 4.52, 3.51, 4.78, 3.34], label=0), + ] + ), + ], categories=['a']) + + with TestDir() as test_dir: + VggFace2Converter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'vgg_face2') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'vgg_face2_dataset') class VggFace2ImporterTest(TestCase): diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index f936eb6746a1..6e3a2db0a565 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -132,10 +132,10 @@ def test_can_detect_voc(self): class VocConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='voc', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_voc_cls(self): class TestExtractor(TestExtractorBase): @@ -401,33 +401,31 @@ def test_can_save_dataset_with_no_subsets(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, annotations=[ - Label(2), - Label(3), - ]), - - DatasetItem(id=2, annotations=[ - Label(3), - ]), + DatasetItem(id=1), + DatasetItem(id=2), ]) - with TestDir() as test_dir: - self._test_save_and_load(TestExtractor(), - partial(VocConverter.convert, label_map='voc'), test_dir) + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task), + test_dir) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id='кириллица с пробелом', annotations=[ - Label(2), - Label(3), - ]), + DatasetItem(id='кириллица с пробелом 1'), + DatasetItem(id='кириллица с пробелом 2', + image=np.ones([4, 5, 3])), ]) - with TestDir() as test_dir: - self._test_save_and_load(TestExtractor(), - partial(VocConverter.convert, label_map='voc'), test_dir) + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task, + save_images=True), + test_dir, require_images=True) def test_can_save_dataset_with_images(self): class TestExtractor(TestExtractorBase): @@ -444,7 +442,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(VocConverter.convert, label_map='voc', save_images=True, tasks=task), - test_dir) + test_dir, require_images=True) def test_dataset_with_voc_labelmap(self): class SrcExtractor(TestExtractorBase): @@ -643,6 +641,23 @@ def __iter__(self): partial(VocConverter.convert, label_map='voc', tasks=task), test_dir) + def test_can_save_and_load_image_with_arbitrary_extension(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ]) + + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task, + save_images=True), + test_dir, require_images=True) + def test_relative_paths(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -657,7 +672,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(VocConverter.convert, label_map='voc', save_images=True, tasks=task), - test_dir) + test_dir, require_images=True) def test_can_save_attributes(self): class TestExtractor(TestExtractorBase): diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index 46163554c90c..0465f5d3f3f2 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -6,6 +6,7 @@ Label, LabelCategories) from datumaro.components.dataset import Dataset from datumaro.plugins.widerface_format import WiderFaceConverter, WiderFaceImporter +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -15,7 +16,7 @@ def test_can_save_and_load(self): DatasetItem(id='1', subset='train', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), Label(0), @@ -23,13 +24,13 @@ def test_can_save_and_load(self): ), DatasetItem(id='2', subset='train', image=np.ones((10, 10, 3)), annotations=[ - Bbox(0, 2, 4, 2, attributes = { + Bbox(0, 2, 4, 2, attributes={ 'blur': '2', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(3, 3, 2, 3, attributes = { + Bbox(3, 3, 2, 3, attributes={ 'blur': '0', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Bbox(2, 1, 2, 3, attributes = { + Bbox(2, 1, 2, 3, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '0', 'invalid': '1'}), Label(1), @@ -38,13 +39,13 @@ def test_can_save_and_load(self): DatasetItem(id='3', subset='val', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 1.1, 5.3, 2.1, attributes = { + Bbox(0, 1.1, 5.3, 2.1, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(0, 2, 3, 2, attributes = { + Bbox(0, 2, 3, 2, attributes={ 'occluded': 'False'}), Bbox(0, 2, 4, 2), - Bbox(0, 7, 3, 2, attributes = { + Bbox(0, 7, 3, 2, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), ] @@ -67,7 +68,7 @@ def test_can_save_dataset_with_no_subsets(self): DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2, label=2), - Bbox(0, 1, 2, 3, label=1, attributes = { + Bbox(0, 1, 2, 3, label=1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), ] @@ -101,17 +102,18 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) parsed_dataset = Dataset.import_from(test_dir, 'wider_face') - compare_datasets(self, source_dataset, parsed_dataset) + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) def test_can_save_dataset_with_non_widerface_attributes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'non-widerface attribute': '0', 'blur': 1, 'invalid': '1'}), - Bbox(1, 1, 2, 2, attributes = { + Bbox(1, 1, 2, 2, attributes={ 'non-widerface attribute': '0'}), ] ), @@ -121,7 +123,7 @@ def test_can_save_dataset_with_non_widerface_attributes(self): DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'blur': '1', 'invalid': '1'}), Bbox(1, 1, 2, 2), ] @@ -134,6 +136,20 @@ def test_can_save_dataset_with_non_widerface_attributes(self): compare_datasets(self, target_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + WiderFaceConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'wider_face') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'widerface_dataset') class WiderFaceImporterTest(TestCase): @@ -145,7 +161,7 @@ def test_can_import(self): DatasetItem(id='0_Parade_image_01', subset='train', image=np.ones((10, 15, 3)), annotations=[ - Bbox(1, 2, 2, 2, attributes = { + Bbox(1, 2, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '0', 'invalid': '0'}), Label(0), @@ -154,10 +170,10 @@ def test_can_import(self): DatasetItem(id='1_Handshaking_image_02', subset='train', image=np.ones((10, 15, 3)), annotations=[ - Bbox(1, 1, 2, 2, attributes = { + Bbox(1, 1, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '0', 'invalid': '0'}), - Bbox(5, 1, 2, 2, attributes = { + Bbox(5, 1, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '0', 'invalid': '0'}), Label(1), @@ -166,13 +182,13 @@ def test_can_import(self): DatasetItem(id='0_Parade_image_03', subset='val', image=np.ones((10, 15, 3)), annotations=[ - Bbox(0, 0, 1, 1, attributes = { + Bbox(0, 0, 1, 1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Bbox(3, 2, 1, 2, attributes = { + Bbox(3, 2, 1, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '0', 'occluded': '1', 'pose': '0', 'invalid': '0'}), - Bbox(5, 6, 1, 1, attributes = { + Bbox(5, 6, 1, 1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), Label(0), diff --git a/tests/test_yolo_format.py b/tests/test_yolo_format.py index f21420e11cd8..5449ba6626de 100644 --- a/tests/test_yolo_format.py +++ b/tests/test_yolo_format.py @@ -106,7 +106,8 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): YoloConverter.convert(source_dataset, test_dir, save_images=True) parsed_dataset = Dataset.import_from(test_dir, 'yolo') - compare_datasets(self, source_dataset, parsed_dataset) + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) def test_relative_paths(self): source_dataset = Dataset.from_iterable([ @@ -116,9 +117,7 @@ def test_relative_paths(self): image=np.ones((2, 6, 3))), DatasetItem(id='subdir2/1', subset='train', image=np.ones((5, 4, 3))), - ], categories={ - AnnotationType.label: LabelCategories(), - }) + ], categories=[]) for save_images in {True, False}: with self.subTest(save_images=save_images): @@ -129,6 +128,20 @@ def test_relative_paths(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', subset='train', + image=Image(path='q/1.JPEG', data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', subset='valid', + image=Image(path='a/b/c/2.bmp', data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + YoloConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'yolo') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset