From 4e2e24a6a6c916b19fe04a4c256816c865b39a76 Mon Sep 17 00:00:00 2001 From: Emily Chun Date: Mon, 5 Apr 2021 19:57:33 +0900 Subject: [PATCH 01/40] Rename 'openvino' plugin to 'openvino_plugin' (#205) Co-authored-by: Jihyeon Yi --- datumaro/plugins/{openvino => openvino_plugin}/README.md | 4 ++-- datumaro/plugins/{openvino => openvino_plugin}/__init__.py | 0 datumaro/plugins/{openvino => openvino_plugin}/launcher.py | 0 .../plugins/{openvino => openvino_plugin}/samples/coco.class | 0 .../{openvino => openvino_plugin}/samples/imagenet.class | 0 .../samples/mobilenet_v2_pytorch_interp.py | 0 .../samples/ssd_face_detection_interp.py | 0 .../samples/ssd_mobilenet_coco_detection_interp.py | 0 .../samples/ssd_person_detection_interp.py | 0 .../samples/ssd_person_vehicle_bike_detection_interp.py | 0 .../samples/ssd_vehicle_detection_interp.py | 0 11 files changed, 2 insertions(+), 2 deletions(-) rename datumaro/plugins/{openvino => openvino_plugin}/README.md (98%) rename datumaro/plugins/{openvino => openvino_plugin}/__init__.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/launcher.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/coco.class (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/imagenet.class (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/mobilenet_v2_pytorch_interp.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/ssd_face_detection_interp.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/ssd_mobilenet_coco_detection_interp.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/ssd_person_detection_interp.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/ssd_person_vehicle_bike_detection_interp.py (100%) rename datumaro/plugins/{openvino => openvino_plugin}/samples/ssd_vehicle_detection_interp.py (100%) diff --git a/datumaro/plugins/openvino/README.md b/datumaro/plugins/openvino_plugin/README.md similarity index 98% rename from datumaro/plugins/openvino/README.md rename to datumaro/plugins/openvino_plugin/README.md index a8f37d3ef9..fc0d904eb9 100644 --- a/datumaro/plugins/openvino/README.md +++ b/datumaro/plugins/openvino_plugin/README.md @@ -67,7 +67,7 @@ You need to implement your own interpreter samples to support the other OpenVINO # Examples # Detection> ssd_mobilenet_v2_coco source /opt/intel/openvino/bin/setupvars.sh - cd datumaro/plugins/openvino + cd datumaro/plugins/openvino_plugin datum create -o proj_ssd_mobilenet_v2_coco_detection datum model add -l openvino -p proj_ssd_mobilenet_v2_coco_detection --copy -- \ --output-layers=do_ExpandDims_conf/sigmoid \ @@ -79,7 +79,7 @@ You need to implement your own interpreter samples to support the other OpenVINO # Classification> mobilenet-v2-pytorch source /opt/intel/openvino/bin/setupvars.sh - cd datumaro/plugins/openvino + cd datumaro/plugins/openvino_plugin datum create -o proj_mobilenet_v2_classification datum model add -l openvino -p proj_mobilenet_v2_classification --copy -- \ -d model/mobilenet-v2-pytorch.xml \ diff --git a/datumaro/plugins/openvino/__init__.py b/datumaro/plugins/openvino_plugin/__init__.py similarity index 100% rename from datumaro/plugins/openvino/__init__.py rename to datumaro/plugins/openvino_plugin/__init__.py diff --git a/datumaro/plugins/openvino/launcher.py b/datumaro/plugins/openvino_plugin/launcher.py similarity index 100% rename from datumaro/plugins/openvino/launcher.py rename to datumaro/plugins/openvino_plugin/launcher.py diff --git a/datumaro/plugins/openvino/samples/coco.class b/datumaro/plugins/openvino_plugin/samples/coco.class similarity index 100% rename from datumaro/plugins/openvino/samples/coco.class rename to datumaro/plugins/openvino_plugin/samples/coco.class diff --git a/datumaro/plugins/openvino/samples/imagenet.class b/datumaro/plugins/openvino_plugin/samples/imagenet.class similarity index 100% rename from datumaro/plugins/openvino/samples/imagenet.class rename to datumaro/plugins/openvino_plugin/samples/imagenet.class diff --git a/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py b/datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py rename to datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py diff --git a/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py b/datumaro/plugins/openvino_plugin/samples/ssd_face_detection_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/ssd_face_detection_interp.py rename to datumaro/plugins/openvino_plugin/samples/ssd_face_detection_interp.py diff --git a/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py b/datumaro/plugins/openvino_plugin/samples/ssd_mobilenet_coco_detection_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py rename to datumaro/plugins/openvino_plugin/samples/ssd_mobilenet_coco_detection_interp.py diff --git a/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py b/datumaro/plugins/openvino_plugin/samples/ssd_person_detection_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/ssd_person_detection_interp.py rename to datumaro/plugins/openvino_plugin/samples/ssd_person_detection_interp.py diff --git a/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py b/datumaro/plugins/openvino_plugin/samples/ssd_person_vehicle_bike_detection_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py rename to datumaro/plugins/openvino_plugin/samples/ssd_person_vehicle_bike_detection_interp.py diff --git a/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py b/datumaro/plugins/openvino_plugin/samples/ssd_vehicle_detection_interp.py similarity index 100% rename from datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py rename to datumaro/plugins/openvino_plugin/samples/ssd_vehicle_detection_interp.py From b14dfa00f469a1e4d64c6a76d82219a3d8352b90 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 5 Apr 2021 14:02:43 +0300 Subject: [PATCH 02/40] Make remap labels more accurate, allow explicit label deletion, add docs, update tests (#203) --- CHANGELOG.md | 19 ++++++++++++++ datumaro/plugins/transforms.py | 37 +++++++++++++++++++--------- tests/test_transforms.py | 45 +++++++++++++++++++++------------- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 560e81d12e..c6a57f84dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- + +### Changed +- + +### Deprecated +- + +### Removed +- + +### Fixed +- Allowed explicit label removal in `remap_labels` transform () + +### Security +- + ## 31/03/2021 - Release v0.1.8 ### Added - diff --git a/datumaro/plugins/transforms.py b/datumaro/plugins/transforms.py index e634794ff7..dfecb25a99 100644 --- a/datumaro/plugins/transforms.py +++ b/datumaro/plugins/transforms.py @@ -18,6 +18,7 @@ ) from datumaro.components.cli_plugin import CliPlugin import datumaro.util.mask_tools as mask_tools +from datumaro.util import parse_str_enum_value, NOTSET from datumaro.util.annotation_util import find_group_leader, find_instances @@ -433,7 +434,22 @@ def transform_item(self, item): class RemapLabels(Transform, CliPlugin): """ Changes labels in the dataset.|n + |n + A label can be:|n + - renamed (and joined with existing) -|n + |s|swhen specified '--label :'|n + - deleted - when specified '--label :' or default action is 'delete'|n + |s|sand the label is not mentioned in the list. When a label|n + |s|sis deleted, all the associated annotations are removed|n + - kept unchanged - when specified '--label :'|n + |s|sor default action is 'keep' and the label is not mentioned in the list|n + Annotations with no label are managed by the default action policy.|n + |n Examples:|n + - Remove the 'person' label (and corresponding annotations):|n + |s|sremap_labels -l person: --default keep|n + - Rename 'person' to 'pedestrian' and 'human' to 'pedestrian', join:|n + |s|sremap_labels -l person:pedestrian -l human:pedestrian --default keep|n - Rename 'person' to 'car' and 'cat' to 'dog', keep 'bus', remove others:|n |s|sremap_labels -l person:car -l bus:bus -l cat:dog --default delete """ @@ -463,9 +479,9 @@ def build_cmdline_parser(cls, **kwargs): def __init__(self, extractor, mapping, default=None): super().__init__(extractor) - assert isinstance(default, (str, self.DefaultAction)) - if isinstance(default, str): - default = self.DefaultAction[default] + default = parse_str_enum_value(default, self.DefaultAction, + self.DefaultAction.keep) + self._default_action = default assert isinstance(mapping, (dict, list)) if isinstance(mapping, list): @@ -503,10 +519,10 @@ def _make_label_id_map(self, src_label_cat, label_mapping, default_action): dst_label_cat = LabelCategories(attributes=src_label_cat.attributes) id_mapping = {} for src_index, src_label in enumerate(src_label_cat.items): - dst_label = label_mapping.get(src_label.name) - if not dst_label and default_action == self.DefaultAction.keep: + dst_label = label_mapping.get(src_label.name, NOTSET) + if dst_label is NOTSET and default_action == self.DefaultAction.keep: dst_label = src_label.name # keep unspecified as is - if not dst_label: + elif not dst_label or dst_label is NOTSET: continue dst_index = dst_label_cat.find(dst_label)[0] @@ -518,7 +534,7 @@ def _make_label_id_map(self, src_label_cat, label_mapping, default_action): if log.getLogger().isEnabledFor(log.DEBUG): log.debug("Label mapping:") for src_id, src_label in enumerate(src_label_cat.items): - if id_mapping.get(src_id): + if id_mapping.get(src_id) is not None: log.debug("#%s '%s' -> #%s '%s'", src_id, src_label.name, id_mapping[src_id], dst_label_cat.items[id_mapping[src_id]].name @@ -535,14 +551,11 @@ def categories(self): def transform_item(self, item): annotations = [] for ann in item.annotations: - if ann.type in { AnnotationType.label, AnnotationType.mask, - AnnotationType.points, AnnotationType.polygon, - AnnotationType.polyline, AnnotationType.bbox - } and ann.label is not None: + if getattr(ann, 'label') is not None: conv_label = self._map_id(ann.label) if conv_label is not None: annotations.append(ann.wrap(label=conv_label)) - else: + elif self._default_action is self.DefaultAction.keep: annotations.append(ann.wrap()) return item.wrap(annotations=annotations) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 5098d03634..1e310c1832 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -336,15 +336,18 @@ def test_remap_labels(self): Bbox(1, 2, 3, 4, label=2), Mask(image=np.array([1]), label=3), - # Should be kept + # Should be deleted Polygon([1, 1, 2, 2, 3, 4], label=4), - PolyLine([1, 3, 4, 2, 5, 6]) + + # Should be kept + PolyLine([1, 3, 4, 2, 5, 6]), + Bbox(4, 3, 2, 1, label=5), ]) ], categories={ AnnotationType.label: LabelCategories.from_iterable( - 'label%s' % i for i in range(5)), + 'label%s' % i for i in range(6)), AnnotationType.mask: MaskCategories( - colormap=mask_tools.generate_colormap(5)), + colormap=mask_tools.generate_colormap(6)), }) dst_dataset = Dataset.from_iterable([ @@ -353,37 +356,45 @@ def test_remap_labels(self): Bbox(1, 2, 3, 4, label=0), Mask(image=np.array([1]), label=1), - Polygon([1, 1, 2, 2, 3, 4], label=2), - PolyLine([1, 3, 4, 2, 5, 6], label=None) + PolyLine([1, 3, 4, 2, 5, 6], label=None), + Bbox(4, 3, 2, 1, label=2), ]), ], categories={ AnnotationType.label: LabelCategories.from_iterable( - ['label0', 'label9', 'label4']), + ['label0', 'label9', 'label5']), AnnotationType.mask: MaskCategories(colormap={ - k: v for k, v in mask_tools.generate_colormap(5).items() - if k in { 0, 1, 3, 4 } + k: v for k, v in mask_tools.generate_colormap(6).items() + if k in { 0, 1, 3, 5 } }) }) actual = transforms.RemapLabels(src_dataset, mapping={ - 'label1': 'label9', - 'label2': 'label0', - 'label3': 'label9', + 'label1': 'label9', # rename & join with new label9 (from label3) + 'label2': 'label0', # rename & join with existing label0 + 'label3': 'label9', # rename & join with new label9 (form label1) + 'label4': '', # delete the label and associated annotations + # 'label5' - unchanged }, default='keep') compare_datasets(self, dst_dataset, actual) def test_remap_labels_delete_unspecified(self): source_dataset = Dataset.from_iterable([ - DatasetItem(id=1, annotations=[ Label(0) ]) - ], categories=['label0']) + DatasetItem(id=1, annotations=[ + Label(0, id=0), # will be removed + Label(1, id=1), + Bbox(1, 2, 3, 4, label=None), + ]) + ], categories=['label0', 'label1']) target_dataset = Dataset.from_iterable([ - DatasetItem(id=1), - ], categories=[]) + DatasetItem(id=1, annotations=[ + Label(0, id=1), + ]), + ], categories=['label1']) actual = transforms.RemapLabels(source_dataset, - mapping={}, default='delete') + mapping={ 'label1': 'label1' }, default='delete') compare_datasets(self, target_dataset, actual) From 70a9005400babad648a5ba035d4393a32bafb815 Mon Sep 17 00:00:00 2001 From: Jihyeon Yi Date: Thu, 8 Apr 2021 18:12:16 +0900 Subject: [PATCH 03/40] Kate/handling multiple attributes and speed up detection split (#207) * better handling multi-attributes for classification_split * handling multi-attributes better for detection * bugfix in calculating required number of images for splitting 2 correct side effect of the changes for re-id split * allow multiple subsets with arbitrary names * rename _is_number to _is_float and improve it --- CHANGELOG.md | 2 +- datumaro/plugins/splitter.py | 270 +++++++++++++++++++++++------------ tests/test_splitter.py | 106 ++++++++++---- 3 files changed, 254 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a57f84dc..a13ea5e8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Changed -- +- Allowed arbitrary subset count and names in classification and detection splitters () ### Deprecated - diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index e4e3b432f7..69240f4ef4 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -4,10 +4,12 @@ import logging as log import numpy as np +from math import gcd from datumaro.components.extractor import (Transform, AnnotationType, DEFAULT_SUBSET_NAME) from datumaro.components.cli_plugin import CliPlugin +from datumaro.util import cast NEAR_ZERO = 1e-7 @@ -33,20 +35,23 @@ def _split_arg(s): raise argparse.ArgumentTypeError() return (parts[0], float(parts[1])) - def __init__(self, dataset, splits, seed): + def __init__(self, dataset, splits, seed, restrict=False): super().__init__(dataset) if splits is None: splits = self._default_split - snames, sratio = self._validate_splits(splits) + snames, sratio, subsets = self._validate_splits(splits, restrict) self._snames = snames self._sratio = sratio self._seed = seed - self._subsets = {"train", "val", "test"} # output subset names + # remove subset name restriction + # regarding https://github.com/openvinotoolkit/datumaro/issues/194 + # self._subsets = {"train", "val", "test"} # output subset names + self._subsets = subsets self._parts = [] self._length = "parent" @@ -70,21 +75,29 @@ def _get_uniq_annotations(dataset): return annotations @staticmethod - def _validate_splits(splits, valid=None): + def _validate_splits(splits, restrict=False): snames = [] ratios = [] - if valid is None: - valid = ["train", "val", "test"] + subsets = set() + valid = ["train", "val", "test"] + # remove subset name restriction + # regarding https://github.com/openvinotoolkit/datumaro/issues/194 for subset, ratio in splits: - assert subset in valid, \ - "Subset name must be one of %s, but got %s" % (valid, subset) + if restrict: + assert subset in valid, \ + "Subset name must be one of %s, got %s" % (valid, subset) assert 0.0 <= ratio and ratio <= 1.0, \ "Ratio is expected to be in the range " \ "[0, 1], but got %s for %s" % (ratio, subset) # ignore near_zero ratio because it may produce partition error. if ratio > NEAR_ZERO: + # handling duplication + if subset in snames: + raise Exception("Subset (%s) is duplicated" % subset) snames.append(subset) ratios.append(float(ratio)) + subsets.add(subset) + ratios = np.array(ratios) total_ratio = np.sum(ratios) @@ -94,15 +107,26 @@ def _validate_splits(splits, valid=None): % (splits, total_ratio) ) - return snames, ratios + return snames, ratios, subsets @staticmethod def _get_required(ratio): - min_value = np.max(ratio) - for i in ratio: - if NEAR_ZERO < i and i < min_value: - min_value = i - required = int(np.around(1.0) / min_value) + if len(ratio) < 2: + return 1 + + for scale in [10, 100]: + farray = np.array(ratio) * scale + iarray = farray.astype(int) + if np.array_equal(iarray, farray): + break + + # find gcd + common_divisor = iarray[0] + for val in iarray[1:]: + common_divisor = gcd(common_divisor, val) + + required = np.sum(np.array(iarray / common_divisor).astype(int)) + return required @staticmethod @@ -129,37 +153,75 @@ def _group_by_attr(items): Returns: by_attributes: dict of { combination-of-attrs : list of index } """ + + # float--> numerical, others(int, string, bool) --> categorical + def _is_float(value): + if isinstance(value, str): + casted = cast(value, float) + if casted is not None: + if cast(casted, str) == value: + return True + return False + elif isinstance(value, float): + cast(value, float) + return True + return False + # group by attributes by_attributes = dict() for idx, ann in items: - attributes = tuple(sorted(ann.attributes.items())) + # ignore numeric attributes + filtered = {} + for k, v in ann.attributes.items(): + if _is_float(v): + continue + filtered[k] = v + attributes = tuple(sorted(filtered.items())) if attributes not in by_attributes: by_attributes[attributes] = [] by_attributes[attributes].append(idx) + return by_attributes def _split_by_attr(self, datasets, snames, ratio, out_splits, - dataset_key=None): + merge_small_classes=True): + + def _split_indice(indice): + sections = self._get_sections(len(indice), ratio) + splits = np.array_split(indice, sections) + for subset, split in zip(snames, splits): + if 0 < len(split): + out_splits[subset].extend(split) + required = self._get_required(ratio) - if dataset_key is None: - dataset_key = "label" - for key, items in datasets.items(): + rest = [] + for _, items in datasets.items(): np.random.shuffle(items) by_attributes = self._group_by_attr(items) - for attributes, indice in by_attributes.items(): - gname = "%s: %s, attrs: %s" % (dataset_key, key, attributes) - splits = self._split_indice(indice, gname, ratio, required) - for subset, split in zip(snames, splits): - if 0 < len(split): - out_splits[subset].extend(split) - - def _split_indice(self, indice, group_name, ratio, required): - filtered_size = len(indice) - if filtered_size < required: - log.warning("Not enough samples for a group, '%s'" % group_name) - sections = self._get_sections(filtered_size, ratio) - splits = np.array_split(indice, sections) - return splits + attr_names = list(by_attributes.keys()) + np.random.shuffle(attr_names) # add randomness + for attr in attr_names: + indice = by_attributes[attr] + quo = len(indice) // required + if quo > 0: + filtered_size = quo * required + _split_indice(indice[:filtered_size]) + rest.extend(indice[filtered_size:]) + else: + rest.extend(indice) + + quo = len(rest) // required + if quo > 0: + filtered_size = quo * required + _split_indice(rest[:filtered_size]) + rest = rest[filtered_size:] + + if not merge_small_classes and len(rest) > 0: + _split_indice(rest) + rest = [] + + if len(rest) > 0: + _split_indice(rest) def _find_split(self, index): for subset_indices, subset in self._parts: @@ -181,7 +243,7 @@ def __iter__(self): class ClassificationSplit(_TaskSpecificSplit): """ - Splits dataset into train/val/test set in class-wise manner. |n + Splits dataset into subsets(train/val/test) in class-wise manner. |n Splits dataset images in the specified ratio, keeping the initial class distribution.|n |n @@ -201,7 +263,6 @@ def __init__(self, dataset, splits, seed=None): dataset : Dataset splits : list A list of (subset(str), ratio(float)) - Subset is expected to be one of ["train", "val", "test"]. The sum of ratios is expected to be 1. seed : int, optional """ @@ -214,6 +275,7 @@ def _split_dataset(self): # 1. group by label by_labels = dict() annotations = self._get_uniq_annotations(self._extractor) + for idx, ann in enumerate(annotations): label = getattr(ann, 'label', None) if label not in by_labels: @@ -290,7 +352,7 @@ def __init__(self, dataset, splits, query=None, if this is not specified, label would be used. seed : int, optional """ - super().__init__(dataset, splits, seed) + super().__init__(dataset, splits, seed, restrict=True) if query is None: query = self._default_query_ratio @@ -300,7 +362,7 @@ def __init__(self, dataset, splits, query=None, "[0, 1], but got %f" % query test_splits = [('test-query', query), ('test-gallery', 1.0 - query)] - # reset output subset names + # remove subset name restriction self._subsets = {"train", "val", "test-gallery", "test-query"} self._test_splits = test_splits self._attr_for_id = attr_for_id @@ -350,7 +412,6 @@ def _split_dataset(self): splits = np.array_split(IDs, sections) testset = {pid: by_id[pid] for pid in splits[0]} trval = {pid: by_id[pid] for pid in splits[1]} - # follow the ratio of datasetitems as possible. # naive heuristic: exchange the best item one by one. expected_count = int(len(self._extractor) * split_ratio[0]) @@ -373,7 +434,7 @@ def _split_dataset(self): test_ratio.append(float(ratio)) self._split_by_attr(testset, test_snames, test_ratio, by_splits, - dataset_key=attr_for_id) + merge_small_classes=False) # 3. split 'trval' into 'train' and 'val' trval_snames = ["train", "val"] @@ -395,7 +456,7 @@ def _split_dataset(self): else: trval_ratio /= total_ratio # normalize self._split_by_attr(trval, trval_snames, trval_ratio, by_splits, - dataset_key=attr_for_id) + merge_small_classes=False) self._set_parts(by_splits) @@ -448,7 +509,7 @@ def _rebalancing(test, trval, expected_count, testset_total): class DetectionSplit(_TaskSpecificSplit): """ - Splits a dataset into train/val/test subsets for detection task, + Splits a dataset into subsets(train/val/test) for detection task, using object annotations as a basis for splitting.|n Tries to produce an image split with the specified ratio, keeping the initial distribution of class objects.|n @@ -476,7 +537,6 @@ def __init__(self, dataset, splits, seed=None): dataset : Dataset splits : list A list of (subset(str), ratio(float)) - Subset is expected to be one of ["train", "val", "test"]. The sum of ratios is expected to be 1. seed : int, optional """ @@ -507,79 +567,105 @@ def _split_dataset(self): by_labels = self._group_by_bbox_labels(self._extractor) # 2. group by attributes - by_combinations = dict() - for label, items in by_labels.items(): + required = self._get_required(sratio) + by_combinations = list() + for _, items in by_labels.items(): by_attributes = self._group_by_attr(items) - for attributes, indice in by_attributes.items(): - gname = "label: %s, attributes: %s" % (label, attributes) - by_combinations[gname] = indice + # merge groups which have too small samples. + attr_names = list(by_attributes.keys()) + np.random.shuffle(attr_names) # add randomless + cluster = [] + minumum = max(required, len(items) * 0.1) # temp solution + for attr in attr_names: + indice = by_attributes[attr] + if len(indice) >= minumum: + by_combinations.append(indice) + else: + cluster.extend(indice) + if len(cluster) >= minumum: + by_combinations.append(cluster) + cluster = [] + if len(cluster) > 0: + by_combinations.append(cluster) + cluster = [] + + total = len(self._extractor) # total number of GT samples per label-attr combinations - n_combs = {k: len(v) for k, v in by_combinations.items()} + n_combs = [len(v) for v in by_combinations] # 3-1. initially count per-image GT samples - scores_all = {} + counts_all = {idx: dict() for idx in range(total)} + for idx_comb, indice in enumerate(by_combinations): + for idx in indice: + if idx_comb not in counts_all[idx]: + counts_all[idx] = {idx_comb: 1} + else: + counts_all[idx][idx_comb] += 1 + init_scores = {} - for idx, _ in enumerate(self._extractor): - counts = {k: v.count(idx) for k, v in by_combinations.items()} - scores_all[idx] = counts - init_scores[idx] = np.sum( - [v / n_combs[k] for k, v in counts.items()] - ) + for idx, counts in counts_all.items(): + norm_sum = 0.0 + for idx_comb, count in counts.items(): + norm_sum += count / n_combs[idx_comb] + init_scores[idx] = norm_sum by_splits = dict() for sname in self._subsets: by_splits[sname] = [] - total = len(self._extractor) target_size = dict() expected = [] # expected numbers of per split GT samples for sname, ratio in zip(subsets, sratio): target_size[sname] = total * ratio - expected.append( - (sname, {k: v * ratio for k, v in n_combs.items()}) - ) + expected.append([sname, np.array(n_combs) * ratio]) # functions for keep the # of annotations not exceed the expected num def compute_penalty(counts, n_combs): p = 0 - for k, v in counts.items(): - p += max(0, (v / n_combs[k]) - 1.0) + for idx_comb, v in counts.items(): + p += max(0, (v / n_combs[idx_comb]) - 1.0) return p def update_nc(counts, n_combs): - for k, v in counts.items(): - n_combs[k] = max(0, n_combs[k] - v) - if n_combs[k] == 0: - n_combs[k] = -1 - return n_combs + for idx_comb, v in counts.items(): + n_combs[idx_comb] = max(0, n_combs[idx_comb] - v) + if n_combs[idx_comb] == 0: + n_combs[idx_comb] = -1 + + by_scores = dict() + for idx, score in init_scores.items(): + if score not in by_scores: + by_scores[score] = [idx] + else: + by_scores[score].append(idx) # 3-2. assign each DatasetItem to a split, one by one - for idx, _ in sorted( - init_scores.items(), key=lambda item: item[1], reverse=True - ): - counts = scores_all[idx] - - # shuffling split order to add randomness - # when two or more splits have the same penalty value - np.random.shuffle(expected) - - pp = [] - for sname, nc in expected: - if target_size[sname] <= len(by_splits[sname]): - # the split has enough images, - # stop adding more images to this split - pp.append(1e08) - else: - # compute penalty based on the number of GT samples - # added in the split - pp.append(compute_penalty(counts, nc)) - - # we push an image to a split with the minimum penalty - midx = np.argmin(pp) - - sname, nc = expected[midx] - by_splits[sname].append(idx) - update_nc(counts, nc) + for score in sorted(by_scores.keys(), reverse=True): + indice = by_scores[score] + np.random.shuffle(indice) # add randomness for the same score + + for idx in indice: + counts = counts_all[idx] + # shuffling split order to add randomness + # when two or more splits have the same penalty value + np.random.shuffle(expected) + + pp = [] + for sname, nc in expected: + if target_size[sname] <= len(by_splits[sname]): + # the split has enough images, + # stop adding more images to this split + pp.append(1e08) + else: + # compute penalty based on the number of GT samples + # added in the split + pp.append(compute_penalty(counts, nc)) + + # we push an image to a split with the minimum penalty + midx = np.argmin(pp) + sname, nc = expected[midx] + by_splits[sname].append(idx) + update_nc(counts, nc) self._set_parts(by_splits) diff --git a/tests/test_splitter.py b/tests/test_splitter.py index 838694a62b..8091cb1f1c 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -44,6 +44,7 @@ def _generate_dataset(self, config): annotations=[ Label(label_id, attributes=attributes) ], + image=np.ones((1, 1, 3)) ) ) else: @@ -51,7 +52,8 @@ def _generate_dataset(self, config): idx += 1 iterable.append( DatasetItem(idx, subset=self._get_subset(idx), - annotations=[Label(label_id)]) + annotations=[Label(label_id)], + image=np.ones((1, 1, 3))) ) categories = {AnnotationType.label: label_cat} dataset = Dataset.from_iterable(iterable, categories) @@ -123,29 +125,37 @@ def test_split_for_classification_single_class_multi_attr(self): config = {"label": {"attrs": attrs, "counts": counts}} source = self._generate_dataset(config) - splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) - - self.assertEqual(84, len(actual.get_subset("train"))) - self.assertEqual(36, len(actual.get_subset("test"))) - - # check stats for train - stat_train = compute_ann_statistics(actual.get_subset("train")) - attr_train = stat_train["annotations"]["labels"]["attributes"] - self.assertEqual(49, attr_train["attr1"]["distribution"]["0"][0]) - self.assertEqual(35, attr_train["attr1"]["distribution"]["1"][0]) - self.assertEqual(28, attr_train["attr2"]["distribution"]["0"][0]) - self.assertEqual(21, attr_train["attr2"]["distribution"]["1"][0]) - self.assertEqual(35, attr_train["attr2"]["distribution"]["2"][0]) - - # check stats for test - stat_test = compute_ann_statistics(actual.get_subset("test")) - attr_test = stat_test["annotations"]["labels"]["attributes"] - self.assertEqual(21, attr_test["attr1"]["distribution"]["0"][0]) - self.assertEqual(15, attr_test["attr1"]["distribution"]["1"][0]) - self.assertEqual(12, attr_test["attr2"]["distribution"]["0"][0]) - self.assertEqual(9, attr_test["attr2"]["distribution"]["1"][0]) - self.assertEqual(15, attr_test["attr2"]["distribution"]["2"][0]) + with self.subTest("zero remainder"): + splits = [("train", 0.7), ("test", 0.3)] + actual = splitter.ClassificationSplit(source, splits) + + self.assertEqual(84, len(actual.get_subset("train"))) + self.assertEqual(36, len(actual.get_subset("test"))) + + # check stats for train + stat_train = compute_ann_statistics(actual.get_subset("train")) + attr_train = stat_train["annotations"]["labels"]["attributes"] + self.assertEqual(49, attr_train["attr1"]["distribution"]["0"][0]) + self.assertEqual(35, attr_train["attr1"]["distribution"]["1"][0]) + self.assertEqual(28, attr_train["attr2"]["distribution"]["0"][0]) + self.assertEqual(21, attr_train["attr2"]["distribution"]["1"][0]) + self.assertEqual(35, attr_train["attr2"]["distribution"]["2"][0]) + + # check stats for test + stat_test = compute_ann_statistics(actual.get_subset("test")) + attr_test = stat_test["annotations"]["labels"]["attributes"] + self.assertEqual(21, attr_test["attr1"]["distribution"]["0"][0]) + self.assertEqual(15, attr_test["attr1"]["distribution"]["1"][0]) + self.assertEqual(12, attr_test["attr2"]["distribution"]["0"][0]) + self.assertEqual(9, attr_test["attr2"]["distribution"]["1"][0]) + self.assertEqual(15, attr_test["attr2"]["distribution"]["2"][0]) + + with self.subTest("non-zero remainder"): + splits = [("train", 0.95), ("test", 0.05)] + actual = splitter.ClassificationSplit(source, splits) + + self.assertEqual(114, len(actual.get_subset("train"))) + self.assertEqual(6, len(actual.get_subset("test"))) def test_split_for_classification_multi_label_with_attr(self): counts = { @@ -221,7 +231,7 @@ def test_split_for_classification_zero_ratio(self): splits = [("train", 0.1), ("val", 0.9), ("test", 0.0)] actual = splitter.ClassificationSplit(source, splits) - + self.assertEqual(1, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(0, len(actual.get_subset("test"))) @@ -263,9 +273,9 @@ def test_split_for_classification_gives_error(self): splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] splitter.ClassificationSplit(source, splits) - with self.subTest("wrong subset name"): - with self.assertRaisesRegex(Exception, "Subset name"): - splits = [("train_", 0.5), ("val", 0.2), ("test", 0.3)] + with self.subTest("duplicated subset name"): + with self.assertRaisesRegex(Exception, "duplicated"): + splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] splitter.ClassificationSplit(source, splits) def test_split_for_reidentification(self): @@ -427,6 +437,11 @@ def test_split_for_reidentification_gives_error(self): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] actual = splitter.ReidentificationSplit(source, splits, -query) + with self.subTest("duplicated subset name"): + with self.assertRaisesRegex(Exception, "duplicated"): + splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] + splitter.ReidentificationSplit(source, splits, query) + with self.subTest("wrong subset name"): with self.assertRaisesRegex(Exception, "Subset name"): splits = [("_train", 0.5), ("val", 0.2), ("test", 0.3)] @@ -650,7 +665,36 @@ def test_split_for_detection_gives_error(self): splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] splitter.DetectionSplit(source, splits) - with self.subTest("wrong subset name"): - with self.assertRaisesRegex(Exception, "Subset name"): - splits = [("train_", 0.5), ("val", 0.2), ("test", 0.3)] + with self.subTest("duplicated subset name"): + with self.assertRaisesRegex(Exception, "duplicated"): + splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] splitter.DetectionSplit(source, splits) + + def test_no_subset_name_and_count_restriction(self): + splits = [("_train", 0.5), ("valid", 0.1), ("valid2", 0.1), + ("test*", 0.2), ("test2", 0.1)] + + with self.subTest("classification"): + config = { + "label1": {"attrs": None, "counts": 10} + } + source = self._generate_dataset(config) + actual = splitter.ClassificationSplit(source, splits) + self.assertEqual(5, len(actual.get_subset("_train"))) + self.assertEqual(1, len(actual.get_subset("valid"))) + self.assertEqual(1, len(actual.get_subset("valid2"))) + self.assertEqual(2, len(actual.get_subset("test*"))) + self.assertEqual(1, len(actual.get_subset("test2"))) + + with self.subTest("detection"): + source, _ = self._generate_detection_dataset( + append_bbox=self._get_append_bbox("cvat"), + with_attr=True, + nimages=10, + ) + actual = splitter.DetectionSplit(source, splits) + self.assertEqual(5, len(actual.get_subset("_train"))) + self.assertEqual(1, len(actual.get_subset("valid"))) + self.assertEqual(1, len(actual.get_subset("valid2"))) + self.assertEqual(2, len(actual.get_subset("test*"))) + self.assertEqual(1, len(actual.get_subset("test2"))) From 43070f4be10644c4ef43c8f7b208288e2309b226 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 8 Apr 2021 13:36:00 +0300 Subject: [PATCH 04/40] Fix voc to coco example (#209) * Fix export filtering * update example in readme --- README.md | 5 +++-- datumaro/cli/contexts/project/__init__.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65aa2817c2..99ebac3664 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ CVAT annotations ---> Publication, statistics etc. # http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar datum convert --input-format voc --input-path \ --output-format coco \ - --filter '/item[annotation/label="cat"]' + --filter '/item[annotation/label="cat"]' \ + -- --reindex 1 # avoid annotation id conflicts ``` - Convert only non-`occluded` annotations from a [CVAT](https://github.com/opencv/cvat) project to TFrecord: @@ -159,7 +160,7 @@ CVAT annotations ---> Publication, statistics etc. - for re-identification task, based on labels, avoiding having same IDs in training and test splits - Sampling a dataset - - analyzes inference result from the given dataset + - analyzes inference result from the given dataset and selects the ‘best’ and the ‘least amount of’ samples for annotation. - Select the sample that best suits model training. - sampling with Entropy based algorithm diff --git a/datumaro/cli/contexts/project/__init__.py b/datumaro/cli/contexts/project/__init__.py index 44e9c82529..64c4a28481 100644 --- a/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/cli/contexts/project/__init__.py @@ -345,7 +345,8 @@ def export_command(args): if args.filter: dataset = dataset.filter(args.filter, **filter_args) - dataset.export(format=args.format, save_dir=dst_dir, **extra_args) + converter = project.env.converters[args.format] + converter.convert(dataset, save_dir=dst_dir, **extra_args) log.info("Project exported to '%s' as '%s'" % (dst_dir, args.format)) From 7b42340bdc84ab9bcc68b4137488f8937c5b66fc Mon Sep 17 00:00:00 2001 From: Kirill Sizov Date: Thu, 8 Apr 2021 17:07:05 +0300 Subject: [PATCH 05/40] Fix export filename for LabelMe format (#200) * change export filename for LabelMe format * Allow simple merge for datasets with no labels * Add a more complex test on relative paths * Support escaping in attributes * update changelog Co-authored-by: Maxim Zhiltsov --- CHANGELOG.md | 4 +- datumaro/components/environment.py | 6 +- datumaro/components/operations.py | 11 +- datumaro/plugins/labelme_format.py | 187 ++++++++++++++++++----------- datumaro/util/__init__.py | 21 +++- tests/test_dataset.py | 25 ++++ tests/test_labelme_format.py | 100 +++++++++------ 7 files changed, 237 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a13ea5e8a2..f815489606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- +- Support for escaping in attribiute values in LabelMe format () ### Changed +- LabelMe format saves dataset items with their relative paths by subsets without changing names () - Allowed arbitrary subset count and names in classification and detection splitters () ### Deprecated @@ -20,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Fixed +- Added support for auto-merging (joining) of datasets with no labels and having labels () - Allowed explicit label removal in `remap_labels` transform () ### Security diff --git a/datumaro/components/environment.py b/datumaro/components/environment.py index 3db6b1b400..c27131a841 100644 --- a/datumaro/components/environment.py +++ b/datumaro/components/environment.py @@ -146,12 +146,12 @@ def __init__(self, config=None): custom = self._load_plugins2(osp.join(env_dir, config.plugins_dir)) select = lambda seq, t: [e for e in seq if issubclass(e, t)] from datumaro.components.converter import Converter - from datumaro.components.extractor import (Importer, SourceExtractor, + from datumaro.components.extractor import (Importer, Extractor, Transform) from datumaro.components.launcher import Launcher self.extractors = PluginRegistry( - builtin=select(builtin, SourceExtractor), - local=select(custom, SourceExtractor) + builtin=select(builtin, Extractor), + local=select(custom, Extractor) ) self.extractors.register(self.PROJECT_EXTRACTOR_NAME, load_project_as_dataset) diff --git a/datumaro/components/operations.py b/datumaro/components/operations.py index 5482b5ed17..6cfdf8e0f4 100644 --- a/datumaro/components/operations.py +++ b/datumaro/components/operations.py @@ -55,10 +55,13 @@ def merge_categories(sources): for source in sources: for cat_type, source_cat in source.items(): existing_cat = categories.setdefault(cat_type, source_cat) - if existing_cat != source_cat: - raise DatumaroError( - "Merging of datasets with different categories is " - "only allowed in 'merge' command.") + if existing_cat != source_cat and len(source_cat) != 0: + if len(existing_cat) == 0: + categories[cat_type] = source_cat + else: + raise DatumaroError( + "Merging of datasets with different categories is " + "only allowed in 'merge' command.") return categories class MergingStrategy(CliPlugin): diff --git a/datumaro/plugins/labelme_format.py b/datumaro/plugins/labelme_format.py index 5580dbc77b..80d9d0d44f 100644 --- a/datumaro/plugins/labelme_format.py +++ b/datumaro/plugins/labelme_format.py @@ -4,15 +4,18 @@ from collections import defaultdict from defusedxml import ElementTree +from functools import partial +from glob import glob, iglob import logging as log import numpy as np import os import os.path as osp -from datumaro.components.extractor import (SourceExtractor, Importer, - DatasetItem, AnnotationType, Mask, Bbox, Polygon, LabelCategories -) +from datumaro.components.extractor import (Extractor, Importer, + DatasetItem, AnnotationType, Mask, Bbox, Polygon, LabelCategories) from datumaro.components.converter import Converter +from datumaro.util import cast, escape, unescape +from datumaro.util.os_util import split_path from datumaro.util.image import Image, save_image from datumaro.util.mask_tools import load_mask, find_mask_bbox @@ -21,70 +24,109 @@ class LabelMePath: MASKS_DIR = 'Masks' IMAGE_EXT = '.jpg' -class LabelMeExtractor(SourceExtractor): - def __init__(self, path, subset=None): + ATTR_IMPORT_ESCAPES = [ + ('\\=', r'%%{eq}%%'), + ('\\"', r'%%{doublequote}%%'), + ('\\,', r'%%{comma}%%'), + ('\\\\', r'%%{backslash}%%'), # keep last + ] + ATTR_EXPORT_ESCAPES = [ + ('\\', '\\\\'), # keep first + ('=', '\\='), + ('"', '\\"'), + (',', '\\,'), + ] + +class LabelMeExtractor(Extractor): + def __init__(self, path): assert osp.isdir(path), path - super().__init__(subset=subset) + super().__init__() - items, categories = self._parse(path) - self._categories = categories - self._items = items + self._items, self._categories, self._subsets = self._parse(path) + self._length = len(self._items) - def _parse(self, path): - categories = { - AnnotationType.label: LabelCategories(attributes={ - 'occluded', 'username' - }) + def _parse(self, dataset_root): + items = [] + subsets = set() + categories = { AnnotationType.label: + LabelCategories(attributes={ 'occluded', 'username' }) } - items = [] - for p in os.listdir(path): - if not p.endswith('.xml'): - continue - root = ElementTree.parse(osp.join(path, p)) + for xml_path in sorted( + glob(osp.join(dataset_root, '**', '*.xml'), recursive=True)): + item_path = osp.relpath(xml_path, dataset_root) + path_parts = split_path(item_path) + subset = '' + if 1 < len(path_parts): + subset = path_parts[0] + item_path = osp.join(*path_parts[1:]) + + root = ElementTree.parse(xml_path) item_id = osp.join(root.find('folder').text or '', - root.find('filename').text) - image_path = osp.join(path, item_id) + root.find('filename').text) or \ + item_path + image_path = osp.join(osp.dirname(xml_path), osp.basename(item_id)) + item_id = osp.splitext(item_id)[0] + image_size = None imagesize_elem = root.find('imagesize') if imagesize_elem is not None: width_elem = imagesize_elem.find('ncols') height_elem = imagesize_elem.find('nrows') image_size = (int(height_elem.text), int(width_elem.text)) + image = Image(path=image_path, size=image_size) - annotations = self._parse_annotations(root, path, categories) + annotations = self._parse_annotations(root, + osp.join(dataset_root, subset), categories) + + items.append(DatasetItem(id=item_id, subset=subset, + image=image, annotations=annotations)) + subsets.add(subset) + return items, categories, subsets + + def _escape(s): + return escape(s, LabelMePath.ATTR_IMPORT_ESCAPES) - items.append(DatasetItem(id=osp.splitext(item_id)[0], - subset=self._subset, image=image, annotations=annotations)) - return items, categories + def _unescape(s): + s = unescape(s, LabelMePath.ATTR_IMPORT_ESCAPES) + s = unescape(s, LabelMePath.ATTR_EXPORT_ESCAPES) + return s @classmethod - def _parse_annotations(cls, xml_root, dataset_root, categories): - def parse_attributes(attr_str): + def _parse_annotations(cls, xml_root, subset_root, categories): + def _parse_attributes(attr_str): parsed = [] if not attr_str: return parsed - for attr in [a.strip() for a in attr_str.split(',') if a.strip()]: + for attr in [a.strip() for a in cls._escape(attr_str).split(',')]: + if not attr: + continue + if '=' in attr: name, value = attr.split('=', maxsplit=1) if value.lower() in {'true', 'false'}: value = value.lower() == 'true' + elif 1 < len(value) and value[0] == '"' and value[-1] == '"': + value = value[1:-1] else: - try: - value = float(value) - except ValueError: - pass - parsed.append((name, value)) + for t in [int, float]: + casted = cast(value, t) + if casted is not None and str(casted) == value: + value = casted + break + if isinstance(value, str): + value = cls._unescape(value) + parsed.append((cls._unescape(name), value)) else: - parsed.append((attr, True)) + parsed.append((cls._unescape(attr), True)) return parsed label_cat = categories[AnnotationType.label] - def get_label_id(label): + def _get_label_id(label): if not label: return None idx, _ = label_cat.find(label) @@ -102,12 +144,12 @@ def get_label_id(label): ann_items = [] - label = get_label_id(obj_elem.find('name').text) + label = _get_label_id(obj_elem.find('name').text) attributes = [] attributes_elem = obj_elem.find('attributes') if attributes_elem is not None and attributes_elem.text: - attributes = parse_attributes(attributes_elem.text) + attributes = _parse_attributes(attributes_elem.text) occluded = False occluded_elem = obj_elem.find('occluded') @@ -156,7 +198,7 @@ def get_label_id(label): user = user_elem.text attributes.append(('username', user)) - mask_path = osp.join(dataset_root, LabelMePath.MASKS_DIR, + mask_path = osp.join(subset_root, LabelMePath.MASKS_DIR, segm_elem.find('mask').text) if not osp.isfile(mask_path): raise Exception("Can't find mask at '%s'" % mask_path) @@ -220,32 +262,28 @@ def get_label_id(label): return image_annotations + def categories(self): + return self._categories + + def __iter__(self): + yield from self._items + class LabelMeImporter(Importer): EXTRACTOR = 'label_me' @classmethod def find_sources(cls, path): - subset_paths = [] + subsets = [] if not osp.isdir(path): return [] - path = osp.normpath(path) - - def has_annotations(d): - return len([p for p in os.listdir(d) if p.endswith('.xml')]) != 0 - - if has_annotations(path): - subset_paths.append({'url': path, 'format': cls.EXTRACTOR}) - else: - for d in os.listdir(path): - subset = d - d = osp.join(path, d) - if osp.isdir(d) and has_annotations(d): - subset_paths.append({'url': d, 'format': cls.EXTRACTOR, - 'options': {'subset': subset} - }) - return subset_paths + try: + next(iglob(osp.join(path, '**', '*.xml'), recursive=True)) + subsets.append({'url': osp.normpath(path), 'format': cls.EXTRACTOR}) + except StopIteration: + pass + return subsets class LabelMeConverter(Converter): @@ -255,18 +293,18 @@ def apply(self): for subset_name, subset in self._extractor.subsets().items(): subset_dir = osp.join(self._save_dir, subset_name) os.makedirs(subset_dir, exist_ok=True) - os.makedirs(osp.join(subset_dir, LabelMePath.MASKS_DIR), - exist_ok=True) - for index, item in enumerate(subset): - self._save_item(item, subset_dir, index) + for item in subset: + self._save_item(item, subset_dir) def _get_label(self, label_id): if label_id is None: return '' return self._extractor.categories()[AnnotationType.label][label_id].name - def _save_item(self, item, subset_dir, index): + _escape = partial(escape, escapes=LabelMePath.ATTR_EXPORT_ESCAPES) + + def _save_item(self, item, subset_dir): from lxml import etree as ET log.debug("Converting item '%s'", item.id) @@ -305,7 +343,7 @@ def _save_item(self, item, subset_dir, index): ET.SubElement(obj_elem, 'deleted').text = '0' ET.SubElement(obj_elem, 'verified').text = '0' ET.SubElement(obj_elem, 'occluded').text = \ - 'yes' if ann.attributes.pop('occluded', '') == True else 'no' + 'yes' if ann.attributes.get('occluded') == True else 'no' ET.SubElement(obj_elem, 'date').text = '' ET.SubElement(obj_elem, 'id').text = str(obj_id) @@ -328,7 +366,7 @@ def _save_item(self, item, subset_dir, index): ET.SubElement(point_elem, 'y').text = '%.2f' % y ET.SubElement(poly_elem, 'username').text = \ - str(ann.attributes.pop('username', '')) + str(ann.attributes.get('username', '')) elif ann.type == AnnotationType.polygon: poly_elem = ET.SubElement(obj_elem, 'polygon') for x, y in zip(ann.points[::2], ann.points[1::2]): @@ -337,13 +375,12 @@ def _save_item(self, item, subset_dir, index): ET.SubElement(point_elem, 'y').text = '%.2f' % y ET.SubElement(poly_elem, 'username').text = \ - str(ann.attributes.pop('username', '')) + str(ann.attributes.get('username', '')) elif ann.type == AnnotationType.mask: - mask_filename = '%s_mask_%s.png' % \ - (item.id.replace('/', '_'), obj_id) + mask_filename = '%s_mask_%s.png' % (item.id, obj_id) save_image(osp.join(subset_dir, LabelMePath.MASKS_DIR, mask_filename), - self._paint_mask(ann.image)) + self._paint_mask(ann.image), create_dir=True) segm_elem = ET.SubElement(obj_elem, 'segm') ET.SubElement(segm_elem, 'mask').text = mask_filename @@ -358,13 +395,21 @@ def _save_item(self, item, subset_dir, index): '%.2f' % (bbox[1] + bbox[3]) ET.SubElement(segm_elem, 'username').text = \ - str(ann.attributes.pop('username', '')) + str(ann.attributes.get('username', '')) else: raise NotImplementedError("Unknown shape type '%s'" % ann.type) attrs = [] for k, v in ann.attributes.items(): - attrs.append('%s=%s' % (k, v)) + if k in { 'username' , 'occluded' }: + continue + if isinstance(v, str): + if cast(v, float) is not None and str(float(v)) == v or \ + cast(v, int) is not None and str(int(v)) == v: + v = f'"{v}"' # add escaping for string values + else: + v = self._escape(v) + attrs.append('%s=%s' % (self._escape(k), v)) ET.SubElement(obj_elem, 'attributes').text = ', '.join(attrs) obj_id += 1 @@ -380,7 +425,11 @@ def _save_item(self, item, subset_dir, index): ET.SubElement(parts_elem, 'hasparts').text = '' ET.SubElement(parts_elem, 'ispartof').text = str(leader_id) - xml_path = osp.join(subset_dir, 'item_%09d.xml' % index) + os.makedirs(osp.join(subset_dir, osp.dirname(image_filename)), + exist_ok=True) + xml_path = osp.join(subset_dir, osp.splitext(image_filename)[0] + '.xml') + if osp.exists(xml_path): + xml_path = osp.join(subset_dir, image_filename + '.xml') with open(xml_path, 'w', encoding='utf-8') as f: xml_data = ET.tostring(root_elem, encoding='unicode', pretty_print=True) diff --git a/datumaro/util/__init__.py b/datumaro/util/__init__.py index ad16d2347d..79190a0fbb 100644 --- a/datumaro/util/__init__.py +++ b/datumaro/util/__init__.py @@ -5,9 +5,10 @@ import attr from contextlib import ExitStack +from distutils.util import strtobool as str_to_bool # pylint: disable=unused-import from functools import partial, wraps from itertools import islice -from distutils.util import strtobool as str_to_bool # pylint: disable=unused-import +from typing import Iterable, Tuple NOTSET = object() @@ -85,6 +86,24 @@ def parse_str_enum_value(value, enum_class, default=NOTSET, (enum_class.__name__, type(value).__name__)) return value +def escape(s: str, escapes: Iterable[Tuple[str, str]]) -> str: + """ + 'escapes' is an iterable of (pattern, substitute) pairs + """ + + for pattern, sub in escapes: + s = s.replace(pattern, sub) + return s + +def unescape(s: str, escapes: Iterable[Tuple[str, str]]) -> str: + """ + 'escapes' is an iterable of (pattern, substitute) pairs + """ + + for pattern, sub in escapes: + s = s.replace(sub, pattern) + return s + def optional_arg_decorator(fn): @wraps(fn) def wrapped_decorator(*args, **kwargs): diff --git a/tests/test_dataset.py b/tests/test_dataset.py index ac5e6b65a6..37f83c6f45 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -86,6 +86,31 @@ def categories(self): compare_datasets(self, TestExtractor(), actual) + def test_can_join_datasets_with_empty_categories(self): + expected = Dataset.from_iterable([ + DatasetItem(1, annotations=[ + Label(0), + Bbox(1, 2, 3, 4), + Caption('hello world'), + ]) + ], categories=['a']) + + src1 = Dataset.from_iterable([ + DatasetItem(1, annotations=[ Bbox(1, 2, 3, 4, label=None) ]) + ], categories=[]) + + src2 = Dataset.from_iterable([ + DatasetItem(1, annotations=[ Label(0) ]) + ], categories=['a']) + + src3 = Dataset.from_iterable([ + DatasetItem(1, annotations=[ Caption('hello world') ]) + ]) + + actual = Dataset.from_extractors(src1, src2, src3) + + compare_datasets(self, expected, actual) + def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ Label(2) ]), diff --git a/tests/test_labelme_format.py b/tests/test_labelme_format.py index 3a514d3d9f..f3d2c3f6ef 100644 --- a/tests/test_labelme_format.py +++ b/tests/test_labelme_format.py @@ -1,12 +1,11 @@ from functools import partial import numpy as np +import os import os.path as osp from unittest import TestCase from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Bbox, Mask, Polygon, LabelCategories -) +from datumaro.components.extractor import (DatasetItem, Bbox, Mask, Polygon) from datumaro.plugins.labelme_format import LabelMeImporter, LabelMeConverter from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, @@ -31,6 +30,8 @@ def test_can_save_and_load(self): 'a1': 'qwe', 'a2': True, 'a3': 123, + 'a4': '42', # must be escaped and recognized as string + 'escaped': 'a,b. = \\= \\\\ " \\" \\, \\', }), Mask(np.array([[0, 1], [1, 0], [1, 1]]), group=2, attributes={ 'username': 'test' }), @@ -40,10 +41,7 @@ def test_can_save_and_load(self): ), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(label) for label in range(10)), - }) + ], categories=['label_' + str(label) for label in range(10)]) target_dataset = Dataset.from_iterable([ DatasetItem(id='dir1/1', subset='train', @@ -60,6 +58,8 @@ def test_can_save_and_load(self): 'a1': 'qwe', 'a2': True, 'a3': 123, + 'a4': '42', + 'escaped': 'a,b. = \\= \\\\ " \\" \\, \\', } ), Mask(np.array([[0, 1], [1, 0], [1, 1]]), group=2, @@ -77,10 +77,7 @@ def test_can_save_and_load(self): ), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable([ - 'label_2', 'label_3']), - }) + ], categories=['label_2', 'label_3']) with TestDir() as test_dir: self._test_save_and_load( @@ -105,38 +102,20 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='train', image=np.ones((16, 16, 3)), - annotations=[ - Polygon([0, 4, 4, 4, 5, 6], label=3, attributes={ - 'occluded': True, - 'a1': 'qwe', - 'a2': True, - 'a3': 123, - }), - ] + annotations=[ Polygon([0, 4, 4, 4, 5, 6], label=3) ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(label) for label in range(10)), - }) + ], categories=['label_' + str(label) for label in range(10)]) target_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='train', image=np.ones((16, 16, 3)), annotations=[ Polygon([0, 4, 4, 4, 5, 6], label=0, id=0, - attributes={ - 'occluded': True, 'username': '', - 'a1': 'qwe', - 'a2': True, - 'a3': 123, - } + attributes={ 'occluded': False, 'username': '' } ), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable([ - 'label_3']), - }) + ], categories=['label_3']) with TestDir() as test_dir: self._test_save_and_load( @@ -144,6 +123,52 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(LabelMeConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + def test_relative_paths(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='1', image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), + + DatasetItem(id='sub/dir3/1', image=np.ones((3, 4, 3)), annotations=[ + Mask(np.array([ + [0, 1, 1, 0], + [0, 1, 1, 0], + [0, 0, 0, 0], + ]), label=1, attributes={ + 'occluded': False, 'username': 'user' + } + ) + ]), + + DatasetItem(id='subdir3/1', subset='a', image=np.ones((5, 4, 3)), + annotations=[ + Bbox(1, 2, 3, 4, label=0, attributes={ + 'occluded': False, 'username': 'user' + }) + ]), + DatasetItem(id='subdir3/1', subset='b', image=np.ones((4, 4, 3))), + ], categories=['label1', 'label2']) + + with TestDir() as test_dir: + self._test_save_and_load(source_dataset, + partial(LabelMeConverter.convert, save_images=True), + test_dir, require_images=True) + + def test_can_save_dataset_to_correct_dir_with_correct_filename(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='dir/a', image=Image(path='dir/a.JPEG', + data=np.zeros((4, 3, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + self._test_save_and_load(dataset, + partial(LabelMeConverter.convert, save_images=True), + test_dir, require_images=True) + + xml_dirpath = osp.join(test_dir, 'default/dir') + self.assertEqual(os.listdir(osp.join(test_dir, 'default')), ['dir']) + self.assertEqual(set(os.listdir(xml_dirpath)), {'a.xml', 'a.JPEG'}) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'labelme_dataset') class LabelMeImporterTest(TestCase): @@ -229,12 +254,9 @@ def test_can_import(self): ), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable([ - 'window', 'license plate', 'o1', - 'q1', 'b1', 'm1', 'hg', - ]), - }) + ], categories=[ + 'window', 'license plate', 'o1', 'q1', 'b1', 'm1', 'hg', + ]) parsed = Dataset.import_from(DUMMY_DATASET_DIR, 'label_me') compare_datasets(self, expected=target_dataset, actual=parsed) \ No newline at end of file From 8b4a99713dd1ee3d6603dfe2990e896f163a46c3 Mon Sep 17 00:00:00 2001 From: Jihyeon Yi Date: Fri, 9 Apr 2021 21:28:02 +0900 Subject: [PATCH 06/40] split unlabeled data into subsets for task-specific splitters (#211) * split unlabeled data into subsets for classification, detection. for re-id, 'not-supported' subsets for this data --- CHANGELOG.md | 1 + datumaro/plugins/splitter.py | 103 +++++++++++++++++++++++++++-------- tests/test_splitter.py | 91 +++++++++++++++---------------- 3 files changed, 127 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f815489606..d9f3b4d690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () - Allowed arbitrary subset count and names in classification and detection splitters () +- Annotation-less dataset elements are now participate in subset splitting () ### Deprecated - diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index 69240f4ef4..c9e19fa8b9 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -49,8 +49,7 @@ def __init__(self, dataset, splits, seed, restrict=False): self._seed = seed # remove subset name restriction - # regarding https://github.com/openvinotoolkit/datumaro/issues/194 - # self._subsets = {"train", "val", "test"} # output subset names + # https://github.com/openvinotoolkit/datumaro/issues/194 self._subsets = subsets self._parts = [] self._length = "parent" @@ -65,14 +64,17 @@ def _set_parts(self, by_splits): @staticmethod def _get_uniq_annotations(dataset): annotations = [] - for item in dataset: + unlabeled_or_multi = [] + + for idx, item in enumerate(dataset): labels = [a for a in item.annotations if a.type == AnnotationType.label] - if len(labels) != 1: - raise Exception("Item '%s' contains %s labels, " - "but exactly one is expected" % (item.id, len(labels))) - annotations.append(labels[0]) - return annotations + if len(labels) == 1: + annotations.append(labels[0]) + else: + unlabeled_or_multi.append(idx) + + return annotations, unlabeled_or_multi @staticmethod def _validate_splits(splits, restrict=False): @@ -80,9 +82,9 @@ def _validate_splits(splits, restrict=False): ratios = [] subsets = set() valid = ["train", "val", "test"] - # remove subset name restriction - # regarding https://github.com/openvinotoolkit/datumaro/issues/194 for subset, ratio in splits: + # remove subset name restriction + # https://github.com/openvinotoolkit/datumaro/issues/194 if restrict: assert subset in valid, \ "Subset name must be one of %s, got %s" % (valid, subset) @@ -143,7 +145,7 @@ def _get_sections(dataset_size, ratio): n_splits[ii] += 1 n_splits[midx] -= 1 sections = np.add.accumulate(n_splits[:-1]) - return sections + return sections, n_splits @staticmethod def _group_by_attr(items): @@ -187,7 +189,7 @@ def _split_by_attr(self, datasets, snames, ratio, out_splits, merge_small_classes=True): def _split_indice(indice): - sections = self._get_sections(len(indice), ratio) + sections, _ = self._get_sections(len(indice), ratio) splits = np.array_split(indice, sections) for subset, split in zip(snames, splits): if 0 < len(split): @@ -223,6 +225,26 @@ def _split_indice(indice): if len(rest) > 0: _split_indice(rest) + def _split_unlabeled(self, unlabeled, by_splits): + """ + split unlabeled data into subsets (detection, classification) + Args: + unlabeled: list of index of unlabeled or multi-labeled data + by_splits: splits up to now + Returns: + by_splits: final splits + """ + dataset_size = len(self._extractor) + _, n_splits = list(self._get_sections(dataset_size, self._sratio)) + counts = [len(by_splits[sname]) for sname in self._snames] + expected = [max(0, v) for v in np.subtract(n_splits, counts)] + sections = np.add.accumulate(expected[:-1]) + np.random.shuffle(unlabeled) + splits = np.array_split(unlabeled, sections) + for subset, split in zip(self._snames, splits): + if 0 < len(split): + by_splits[subset].extend(split) + def _find_split(self, index): for subset_indices, subset in self._parts: if index in subset_indices: @@ -248,7 +270,8 @@ class ClassificationSplit(_TaskSpecificSplit): distribution.|n |n Notes:|n - - Each image is expected to have only one Label|n + - Each image is expected to have only one Label. Unlabeled or + multi-labeled images will be split into subsets randomly. |n - If Labels also have attributes, also splits by attribute values.|n - If there is not enough images in some class or attributes group, the split ratio can't be guaranteed.|n @@ -274,7 +297,7 @@ def _split_dataset(self): # support only single label for a DatasetItem # 1. group by label by_labels = dict() - annotations = self._get_uniq_annotations(self._extractor) + annotations, unlabeled = self._get_uniq_annotations(self._extractor) for idx, ann in enumerate(annotations): label = getattr(ann, 'label', None) @@ -288,6 +311,12 @@ def _split_dataset(self): # 2. group by attributes self._split_by_attr(by_labels, self._snames, self._sratio, by_splits) + + # 3. split unlabeled data + if len(unlabeled) > 0: + self._split_unlabeled(unlabeled, by_splits) + + # 4. set parts self._set_parts(by_splits) @@ -310,7 +339,8 @@ class ReidentificationSplit(_TaskSpecificSplit): 'train', 'val', 'test-gallery' and 'test-query'. |n |n Notes:|n - - Each image is expected to have a single Label|n + - Each image is expected to have a single Label. Unlabeled or multi-labeled + images will be split into 'not-supported'.|n - Object ID can be described by Label, or by attribute (--attr parameter)|n - The splits of the test set are controlled by '--query' parameter. |n |s|sGallery ratio would be 1.0 - query.|n @@ -377,7 +407,7 @@ def _split_dataset(self): # group by ID(attr_for_id) by_id = dict() - annotations = self._get_uniq_annotations(dataset) + annotations, unlabeled = self._get_uniq_annotations(dataset) if attr_for_id is None: # use label for idx, ann in enumerate(annotations): ID = getattr(ann, 'label', None) @@ -408,7 +438,7 @@ def _split_dataset(self): split_ratio = np.array([test, 1.0 - test]) IDs = list(by_id.keys()) np.random.shuffle(IDs) - sections = self._get_sections(len(IDs), split_ratio) + sections, _ = self._get_sections(len(IDs), split_ratio) splits = np.array_split(IDs, sections) testset = {pid: by_id[pid] for pid in splits[0]} trval = {pid: by_id[pid] for pid in splits[1]} @@ -458,6 +488,11 @@ def _split_dataset(self): self._split_by_attr(trval, trval_snames, trval_ratio, by_splits, merge_small_classes=False) + # split unlabeled data into 'not-supported'. + if len(unlabeled) > 0: + self._subsets.add("not-supported") + by_splits["not-supported"] = unlabeled + self._set_parts(by_splits) @staticmethod @@ -506,6 +541,20 @@ def _rebalancing(test, trval, expected_count, testset_total): test[id_trval] = trval.pop(id_trval) trval[id_test] = test.pop(id_test) + def get_subset(self, name): + # lazy splitting + if self._initialized is False: + self._split_dataset() + self._initialized = True + return super().get_subset(name) + + def subsets(self): + # lazy splitting + if self._initialized is False: + self._split_dataset() + self._initialized = True + return super().subsets() + class DetectionSplit(_TaskSpecificSplit): """ @@ -545,18 +594,20 @@ def __init__(self, dataset, splits, seed=None): @staticmethod def _group_by_bbox_labels(dataset): by_labels = dict() + unlabeled = [] for idx, item in enumerate(dataset): bbox_anns = [a for a in item.annotations if a.type == AnnotationType.bbox] - assert 0 < len(bbox_anns), \ - "Expected more than one bbox annotation in the dataset" + if len(bbox_anns) == 0: + unlabeled.append(idx) + continue for ann in bbox_anns: label = getattr(ann, 'label', None) if label not in by_labels: by_labels[label] = [(idx, ann)] else: by_labels[label].append((idx, ann)) - return by_labels + return by_labels, unlabeled def _split_dataset(self): np.random.seed(self._seed) @@ -564,7 +615,7 @@ def _split_dataset(self): subsets, sratio = self._snames, self._sratio # 1. group by bbox label - by_labels = self._group_by_bbox_labels(self._extractor) + by_labels, unlabeled = self._group_by_bbox_labels(self._extractor) # 2. group by attributes required = self._get_required(sratio) @@ -595,7 +646,11 @@ def _split_dataset(self): n_combs = [len(v) for v in by_combinations] # 3-1. initially count per-image GT samples - counts_all = {idx: dict() for idx in range(total)} + counts_all = {} + for idx in range(total): + if idx not in unlabeled: + counts_all[idx] = dict() + for idx_comb, indice in enumerate(by_combinations): for idx in indice: if idx_comb not in counts_all[idx]: @@ -668,4 +723,8 @@ def update_nc(counts, n_combs): by_splits[sname].append(idx) update_nc(counts, nc) + # split unlabeled data + if len(unlabeled) > 0: + self._split_unlabeled(unlabeled, by_splits) + self._set_parts(by_splits) diff --git a/tests/test_splitter.py b/tests/test_splitter.py index 8091cb1f1c..351162b9c9 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -236,29 +236,27 @@ def test_split_for_classification_zero_ratio(self): self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(0, len(actual.get_subset("test"))) - def test_split_for_classification_gives_error(self): + def test_split_for_classification_unlabeled(self): with self.subTest("no label"): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[]), - DatasetItem(2, annotations=[]), - ], categories=["a", "b", "c"]) + iterable = [DatasetItem(i, annotations=[]) for i in range(10)] + source = Dataset.from_iterable(iterable, categories=["a", "b"]) + splits = [("train", 0.7), ("test", 0.3)] + actual = splitter.ClassificationSplit(source, splits) - with self.assertRaisesRegex(Exception, "exactly one is expected"): - splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) - len(actual.get_subset("train")) + self.assertEqual(7, len(actual.get_subset("train"))) + self.assertEqual(3, len(actual.get_subset("test"))) with self.subTest("multi label"): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[Label(0), Label(1)]), - DatasetItem(2, annotations=[Label(0), Label(2)]), - ], categories=["a", "b", "c"]) + anns = [Label(0), Label(1)] + iterable = [DatasetItem(i, annotations=anns) for i in range(10)] + source = Dataset.from_iterable(iterable, categories=["a", "b"]) + splits = [("train", 0.7), ("test", 0.3)] + actual = splitter.ClassificationSplit(source, splits) - with self.assertRaisesRegex(Exception, "exactly one is expected"): - splits = [("train", 0.7), ("test", 0.3)] - splitter.ClassificationSplit(source, splits) - len(actual.get_subset("train")) + self.assertEqual(7, len(actual.get_subset("train"))) + self.assertEqual(3, len(actual.get_subset("test"))) + def test_split_for_classification_gives_error(self): source = Dataset.from_iterable([ DatasetItem(1, annotations=[Label(0)]), DatasetItem(2, annotations=[Label(1)]), @@ -396,30 +394,27 @@ def test_split_for_reidentification_rebalance(self): self.assertEqual(90, len(actual.get_subset("test-gallery"))) self.assertEqual(120, len(actual.get_subset("test-query"))) - def test_split_for_reidentification_gives_error(self): - query = 0.4 / 0.7 # valid query ratio + def test_split_for_reidentification_unlabeled(self): + query = 0.5 with self.subTest("no label"): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[]), - DatasetItem(2, annotations=[]), - ], categories=["a", "b", "c"]) + iterable = [DatasetItem(i, annotations=[]) for i in range(10)] + source = Dataset.from_iterable(iterable, categories=["a", "b"]) + splits = [("train", 0.6), ("test", 0.4)] + actual = splitter.ReidentificationSplit(source, splits, query) + self.assertEqual(10, len(actual.get_subset("not-supported"))) - with self.assertRaisesRegex(Exception, "exactly one is expected"): - splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.ReidentificationSplit(source, splits, query) - len(actual.get_subset("train")) + with self.subTest("multi label"): + anns = [Label(0), Label(1)] + iterable = [DatasetItem(i, annotations=anns) for i in range(10)] + source = Dataset.from_iterable(iterable, categories=["a", "b"]) + splits = [("train", 0.6), ("test", 0.4)] + actual = splitter.ReidentificationSplit(source, splits, query) - with self.subTest(msg="multi label"): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[Label(0), Label(1)]), - DatasetItem(2, annotations=[Label(0), Label(2)]), - ], categories=["a", "b", "c"]) + self.assertEqual(10, len(actual.get_subset("not-supported"))) - with self.assertRaisesRegex(Exception, "exactly one is expected"): - splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.ReidentificationSplit(source, splits, query) - len(actual.get_subset("train")) + def test_split_for_reidentification_gives_error(self): + query = 0.4 / 0.7 # valid query ratio counts = {i: (i % 3 + 1) * 7 for i in range(10)} config = {"person": {"attrs": ["PID"], "counts": counts}} @@ -638,18 +633,22 @@ def test_split_for_detection(self): list(r1.get_subset("test")), list(r3.get_subset("test")) ) - def test_split_for_detection_gives_error(self): - with self.subTest(msg="bbox annotation"): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[Label(0), Label(1)]), - DatasetItem(2, annotations=[Label(0), Label(2)]), - ], categories=["a", "b", "c"]) + def test_split_for_detection_with_unlabeled(self): + source, _ = self._generate_detection_dataset( + append_bbox=self._get_append_bbox("cvat"), + with_attr=True, + nimages=10, + ) + for i in range(10): + source.put(DatasetItem(i + 10, annotations={})) - with self.assertRaisesRegex(Exception, "more than one bbox"): - splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.DetectionSplit(source, splits) - len(actual.get_subset("train")) + splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] + actual = splitter.DetectionSplit(source, splits) + self.assertEqual(10, len(actual.get_subset("train"))) + self.assertEqual(4, len(actual.get_subset("val"))) + self.assertEqual(6, len(actual.get_subset("test"))) + def test_split_for_detection_gives_error(self): source, _ = self._generate_detection_dataset( append_bbox=self._get_append_bbox("cvat"), with_attr=True, From 989f990ec498e2d011a956a285bea5d0bd01c459 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 10 Apr 2021 08:52:01 +0300 Subject: [PATCH 07/40] Fix image ext on saving in cvat format (#214) * fix image saving in cvat format * update changelog --- CHANGELOG.md | 1 + datumaro/plugins/cvat_format/converter.py | 2 +- tests/test_cvat_format.py | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9f3b4d690..9963e4707e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Added support for auto-merging (joining) of datasets with no labels and having labels () - Allowed explicit label removal in `remap_labels` transform () +- Image extension in CVAT format export () ### Security - diff --git a/datumaro/plugins/cvat_format/converter.py b/datumaro/plugins/cvat_format/converter.py index 5a9f64469b..4b8c07982b 100644 --- a/datumaro/plugins/cvat_format/converter.py +++ b/datumaro/plugins/cvat_format/converter.py @@ -164,7 +164,7 @@ def _write_item(self, item, index): if not self._context._reindex: index = cast(item.attributes.get('frame'), int, index) image_info = OrderedDict([ ("id", str(index)), ]) - filename = item.id + CvatPath.IMAGE_EXT + filename = self._context._make_image_filename(item) image_info["name"] = filename if item.has_image: size = item.image.size diff --git a/tests/test_cvat_format.py b/tests/test_cvat_format.py index 5b2c60e130..12d4b82205 100644 --- a/tests/test_cvat_format.py +++ b/tests/test_cvat_format.py @@ -326,6 +326,10 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): self._test_save_and_load(expected, partial(CvatConverter.convert, save_images=True), test_dir, require_images=True) + self.assertTrue(osp.isfile( + osp.join(test_dir, 'images', 'q', '1.JPEG'))) + self.assertTrue(osp.isfile( + osp.join(test_dir, 'images', 'a', 'b', 'c', '2.bmp'))) def test_preserve_frame_ids(self): expected_dataset = Dataset.from_iterable([ From d549952c8b023fd7cf8b3ae121775f8ec5d9ed16 Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Tue, 13 Apr 2021 17:42:40 +0300 Subject: [PATCH 08/40] Label "face" for bounding boxes in Wider Face (#215) * add face label * update changelog --- CHANGELOG.md | 1 + datumaro/plugins/widerface_format.py | 37 +++++++++++-------- tests/test_widerface_format.py | 55 ++++++++++++---------------- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9963e4707e..ef2dd699a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for auto-merging (joining) of datasets with no labels and having labels () - Allowed explicit label removal in `remap_labels` transform () - Image extension in CVAT format export () +- Added a label "face" for bounding boxes in Wider Face () ### Security - diff --git a/datumaro/plugins/widerface_format.py b/datumaro/plugins/widerface_format.py index f5e0008f60..96796b09f3 100644 --- a/datumaro/plugins/widerface_format.py +++ b/datumaro/plugins/widerface_format.py @@ -21,6 +21,7 @@ class WiderFacePath: IMAGES_DIR_NO_LABEL = 'no_label' BBOX_ATTRIBUTES = ['blur', 'expression', 'illumination', 'occluded', 'pose', 'invalid'] + DEFAULT_LABEL = 'face' class WiderFaceExtractor(SourceExtractor): def __init__(self, path, subset=None): @@ -40,13 +41,13 @@ def __init__(self, path, subset=None): def _load_categories(self): label_cat = LabelCategories() - path = osp.join(self._dataset_dir, WiderFacePath.LABELS_FILE) if osp.isfile(path): with open(path, encoding='utf-8') as labels_file: for line in labels_file: label_cat.add(line.strip()) else: + label_cat.add(WiderFacePath.DEFAULT_LABEL) subset_path = osp.join(self._dataset_dir, WiderFacePath.SUBSET_DIR + self._subset, WiderFacePath.IMAGES_DIR) @@ -56,12 +57,15 @@ def _load_categories(self): images_dir != WiderFacePath.IMAGES_DIR_NO_LABEL: if '--' in images_dir: images_dir = images_dir.split('--')[1] - label_cat.add(images_dir) - + if images_dir != WiderFacePath.DEFAULT_LABEL: + label_cat.add(images_dir) + if len(label_cat) == 1: + label_cat = LabelCategories() return { AnnotationType.label: label_cat } def _load_items(self, path): items = {} + label_categories = self._categories[AnnotationType.label] with open(path, 'r', encoding='utf-8') as f: lines = f.readlines() @@ -73,6 +77,7 @@ def _load_items(self, path): for line_idx in line_ids: image_path = lines[line_idx].strip() item_id = osp.splitext(image_path)[0] + item_id = item_id.replace('\\', '/') image_path = osp.join(self._dataset_dir, WiderFacePath.SUBSET_DIR + self._subset, @@ -84,9 +89,9 @@ def _load_items(self, path): if '--' in label_name: label_name = label_name.split('--')[1] if label_name != WiderFacePath.IMAGES_DIR_NO_LABEL: - label = \ - self._categories[AnnotationType.label].find(label_name)[0] - annotations.append(Label(label=label)) + label = label_categories.find(label_name)[0] + if label != None: + annotations.append(Label(label=label)) item_id = item_id[len(item_id.split('/')[0]) + 1:] items[item_id] = DatasetItem(id=item_id, subset=self._subset, @@ -101,21 +106,22 @@ def _load_items(self, path): for bbox in bbox_lines: bbox_list = bbox.split() if 4 <= len(bbox_list): - attributes = {} - label = None + label = label_categories.find(WiderFacePath.DEFAULT_LABEL)[0] if len(bbox_list) == 5 or len(bbox_list) == 11: - if len(bbox_list) == 5: - label_name = bbox_list[4] - else: - label_name = bbox_list[10] - label = \ - self._categories[AnnotationType.label].find(label_name)[0] + label_name = bbox_list[-1] + label = label_categories.find(label_name)[0] + if label == None and len(label_categories) == 0: + label_categories.add(WiderFacePath.DEFAULT_LABEL) + label = label_categories.find(WiderFacePath.DEFAULT_LABEL)[0] + + attributes = {} if 10 <= len(bbox_list): i = 4 for attr in WiderFacePath.BBOX_ATTRIBUTES: if bbox_list[i] != '-': attributes[attr] = bbox_list[i] i += 1 + annotations.append(Bbox( float(bbox_list[0]), float(bbox_list[1]), float(bbox_list[2]), float(bbox_list[3]), @@ -180,7 +186,8 @@ def apply(self): wider_attr += '- ' if 0 < attr_counter: wider_annotation += wider_attr - if bbox.label is not None: + if label_categories[bbox.label].name != WiderFacePath.DEFAULT_LABEL and \ + bbox.label is not None: wider_annotation += '%s' % label_categories[bbox.label].name wider_annotation += '\n' diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index 0465f5d3f3..8e2586999e 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -15,47 +15,44 @@ def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes={ + Bbox(0, 2, 4, 2, label=0), + Bbox(0, 1, 2, 3, label=0, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Label(0), + Label(1), ] ), DatasetItem(id='2', subset='train', image=np.ones((10, 10, 3)), annotations=[ - Bbox(0, 2, 4, 2, attributes={ + Bbox(0, 2, 4, 2, label=0, attributes={ 'blur': '2', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(3, 3, 2, 3, attributes={ + Bbox(3, 3, 2, 3, label=0, attributes={ 'blur': '0', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Bbox(2, 1, 2, 3, attributes={ + Bbox(2, 1, 2, 3, label=0, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '0', 'invalid': '1'}), - Label(1), + Label(2), ] ), DatasetItem(id='3', subset='val', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 1.1, 5.3, 2.1, attributes={ + Bbox(0, 1.1, 5.3, 2.1, label=0, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(0, 2, 3, 2, attributes={ + Bbox(0, 2, 3, 2, label=0, attributes={ 'occluded': 'False'}), - Bbox(0, 2, 4, 2), - Bbox(0, 7, 3, 2, attributes={ + Bbox(0, 2, 4, 2, label=0), + Bbox(0, 7, 3, 2, label=0, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), ] ), DatasetItem(id='4', subset='val', image=np.ones((8, 8, 3))), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(i) for i in range(3)), - }) + ], categories=['face', 'label_0', 'label_1']) with TestDir() as test_dir: WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) @@ -73,10 +70,7 @@ def test_can_save_dataset_with_no_subsets(self): 'occluded': '0', 'pose': '2', 'invalid': '0'}), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(i) for i in range(3)), - }) + ], categories=['face', 'label_0', 'label_1']) with TestDir() as test_dir: WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) @@ -88,15 +82,12 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 1, 2, 3, label=1, attributes = { + Bbox(0, 1, 2, 3, label=0, attributes = { 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), ] ), - ], categories={ - AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(i) for i in range(3)), - }) + ], categories=['face']) with TestDir() as test_dir: WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) @@ -109,26 +100,26 @@ def test_can_save_dataset_with_non_widerface_attributes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes={ + Bbox(0, 2, 4, 2, label=0), + Bbox(0, 1, 2, 3, label=0, attributes={ 'non-widerface attribute': '0', 'blur': 1, 'invalid': '1'}), - Bbox(1, 1, 2, 2, attributes={ + Bbox(1, 1, 2, 2, label=0, attributes={ 'non-widerface attribute': '0'}), ] ), - ], categories=[]) + ], categories=['face']) target_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes={ + Bbox(0, 2, 4, 2, label=0), + Bbox(0, 1, 2, 3, label=0, attributes={ 'blur': '1', 'invalid': '1'}), - Bbox(1, 1, 2, 2), + Bbox(1, 1, 2, 2, label=0), ] ), - ], categories=[]) + ], categories=['face']) with TestDir() as test_dir: WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) From b3eaf4a5035ad37c4cf591643b5d0f4b18f64992 Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Wed, 14 Apr 2021 10:35:38 +0300 Subject: [PATCH 09/40] Adding "difficult", "truncated", "occluded" attributes when converting to Pascal VOC if they are not present (#216) * remove check for 'difficult' attribute * remove check for 'truncated' and 'occluded' attributes * update changelog --- CHANGELOG.md | 1 + datumaro/plugins/voc_format/converter.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef2dd699a1..f0e966c5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allowed explicit label removal in `remap_labels` transform () - Image extension in CVAT format export () - Added a label "face" for bounding boxes in Wider Face () +- Allowed adding "difficult", "truncated", "occluded" attributes when converting to Pascal VOC if these attributes are not present () ### Security - diff --git a/datumaro/plugins/voc_format/converter.py b/datumaro/plugins/voc_format/converter.py index abb109ff9a..4a5d36b9f6 100644 --- a/datumaro/plugins/voc_format/converter.py +++ b/datumaro/plugins/voc_format/converter.py @@ -236,20 +236,12 @@ def save_subsets(self): ET.SubElement(obj_elem, 'pose').text = \ str(attr['pose']) - if 'truncated' in attr: - truncated = _convert_attr('truncated', attr, int, 0) - ET.SubElement(obj_elem, 'truncated').text = \ - '%d' % truncated - - if 'difficult' in attr: - difficult = _convert_attr('difficult', attr, int, 0) - ET.SubElement(obj_elem, 'difficult').text = \ - '%d' % difficult - - if 'occluded' in attr: - occluded = _convert_attr('occluded', attr, int, 0) - ET.SubElement(obj_elem, 'occluded').text = \ - '%d' % occluded + ET.SubElement(obj_elem, 'truncated').text = \ + '%d' % _convert_attr('truncated', attr, int, 0) + ET.SubElement(obj_elem, 'occluded').text = \ + '%d' % _convert_attr('occluded', attr, int, 0) + ET.SubElement(obj_elem, 'difficult').text = \ + '%d' % _convert_attr('difficult', attr, int, 0) bbox = obj.get_bbox() if bbox is not None: From 2f9614a10cb035a745bf35a6f40100339e511987 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 20 Apr 2021 11:41:41 +0300 Subject: [PATCH 10/40] Ignore empty lines in YOLO annotations (#221) * Ignore empty lines in yolo annotations * Add type hints for image class, catch image opening errors in image.size * update changelog --- CHANGELOG.md | 1 + datumaro/components/operations.py | 5 ++++ datumaro/plugins/yolo_format/extractor.py | 6 ++-- datumaro/util/image.py | 34 +++++++++++++---------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0e966c5fc..cf0a07e515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Image extension in CVAT format export () - Added a label "face" for bounding boxes in Wider Face () - Allowed adding "difficult", "truncated", "occluded" attributes when converting to Pascal VOC if these attributes are not present () +- Empty lines in YOLO annotations are ignored () ### Security - diff --git a/datumaro/components/operations.py b/datumaro/components/operations.py index 6cfdf8e0f4..db3c9c3889 100644 --- a/datumaro/components/operations.py +++ b/datumaro/components/operations.py @@ -957,6 +957,11 @@ def mean_std(dataset): var = lambda i, s: s[i][1] for i, item in enumerate(dataset): + size = item.image.size + if size is None: + log.warning("Item %s: can't detect image size, " + "the image will be skipped from pixel statistics", item.id) + continue counts[i] = np.prod(item.image.size) image = item.image.data diff --git a/datumaro/plugins/yolo_format/extractor.py b/datumaro/plugins/yolo_format/extractor.py index 33ab8eb7ff..77a7d958b2 100644 --- a/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/plugins/yolo_format/extractor.py @@ -92,7 +92,7 @@ def __init__(self, config_path, image_info=None): with open(list_path, 'r', encoding='utf-8') as f: subset.items = OrderedDict( (self.name_from_path(p), self.localize_path(p)) - for p in f + for p in f if p.strip() ) subsets[subset_name] = subset @@ -176,7 +176,9 @@ def _load_categories(names_path): with open(names_path, 'r', encoding='utf-8') as f: for label in f: - label_categories.add(label.strip()) + label = label.strip() + if label: + label_categories.add(label) return label_categories diff --git a/datumaro/util/image.py b/datumaro/util/image.py index 9500232263..17e2a0d0ba 100644 --- a/datumaro/util/image.py +++ b/datumaro/util/image.py @@ -7,19 +7,21 @@ from enum import Enum from io import BytesIO -from typing import Iterator, Iterable, Union +from typing import Any, Callable, Iterator, Iterable, Optional, Tuple, Union import numpy as np import os import os.path as osp _IMAGE_BACKENDS = Enum('_IMAGE_BACKENDS', ['cv2', 'PIL']) _IMAGE_BACKEND = None +_image_loading_errors = (FileNotFoundError, ) try: import cv2 _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2 except ImportError: import PIL _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL + _image_loading_errors = (*_image_loading_errors, PIL.UnidentifiedImageError) from datumaro.util.image_cache import ImageCache as _ImageCache from datumaro.util.os_util import walk @@ -33,6 +35,8 @@ def load_image(path, dtype=np.float32): if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2: import cv2 image = cv2.imread(path, cv2.IMREAD_UNCHANGED) + if image is None: + raise FileNotFoundError("Can't open image: %s" % path) image = image.astype(dtype) elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL: from PIL import Image @@ -43,8 +47,6 @@ def load_image(path, dtype=np.float32): else: raise NotImplementedError() - if image is None: - raise ValueError("Can't open image '%s'" % path) assert len(image.shape) in {2, 3} if len(image.shape) == 3: assert image.shape[2] in {3, 4} @@ -227,15 +229,16 @@ def __hash__(self): return hash((id(self), self.path, self.loader)) class Image: - def __init__(self, data=None, path=None, loader=None, cache=None, - size=None): - assert size is None or len(size) == 2 + def __init__(self, data: Union[None, Callable, np.ndarray] = None, + path: Optional[str] = None, loader: Optional[Callable] = None, + size: Optional[Tuple[int, int]] = None, cache: Any = None): + assert size is None or len(size) == 2, size if size is not None: assert len(size) == 2 and 0 < size[0] and 0 < size[1], size size = tuple(size) self._size = size # (H, W) - assert path is None or isinstance(path, str) + assert path is None or isinstance(path, str), path if path is None: path = '' elif path: @@ -254,15 +257,15 @@ def __init__(self, data=None, path=None, loader=None, cache=None, self._size = data.shape[:2] @property - def path(self): + def path(self) -> str: return self._path @property - def ext(self): + def ext(self) -> str: return osp.splitext(osp.basename(self.path))[1] @property - def data(self): + def data(self) -> np.ndarray: if callable(self._data): data = self._data() else: @@ -273,17 +276,20 @@ def data(self): return data @property - def has_data(self): + def has_data(self) -> bool: return self._data is not None @property - def has_size(self): + def has_size(self) -> bool: return self._size is not None or isinstance(self._data, np.ndarray) @property - def size(self): + def size(self) -> Optional[Tuple[int, int]]: if self._size is None: - data = self.data + try: + data = self.data + except _image_loading_errors: + return None if data is not None: self._size = data.shape[:2] return self._size From 4279bc8a7fa6cc4e0d13826f1caa2cf410078d01 Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Tue, 20 Apr 2021 18:29:51 +0300 Subject: [PATCH 11/40] Classification task in LFW dataset format (#222) * add classification * update changelog * update documentation --- CHANGELOG.md | 1 + README.md | 2 +- datumaro/plugins/lfw_format.py | 242 +++++++++++++++++------ docs/user_manual.md | 2 +- tests/assets/lfw_dataset/test/people.txt | 3 + tests/test_lfw_format.py | 176 ++++++++--------- 6 files changed, 271 insertions(+), 155 deletions(-) create mode 100644 tests/assets/lfw_dataset/test/people.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0a07e515..f4dffa8b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - LabelMe format saves dataset items with their relative paths by subsets without changing names () - Allowed arbitrary subset count and names in classification and detection splitters () - Annotation-less dataset elements are now participate in subset splitting () +- Classification task in LFW dataset format () ### Deprecated - diff --git a/README.md b/README.md index 99ebac3664..3fba9e10fb 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ CVAT annotations ---> Publication, statistics etc. - [LabelMe](http://labelme.csail.mit.edu/Release3.0) - [ICDAR13/15](https://rrc.cvc.uab.es/?ch=2) (`word_recognition`, `text_localization`, `text_segmentation`) - [Market-1501](https://www.aitribune.com/dataset/2018051063) (`person re-identification`) - - [LFW](http://vis-www.cs.umass.edu/lfw/) (`person re-identification`, `landmarks`) + - [LFW](http://vis-www.cs.umass.edu/lfw/) (`classification`, `person re-identification`, `landmarks`) - Dataset building - Merging multiple datasets into one - Dataset filtering by a custom criteria: diff --git a/datumaro/plugins/lfw_format.py b/datumaro/plugins/lfw_format.py index 5799ad87e5..c4806647cb 100644 --- a/datumaro/plugins/lfw_format.py +++ b/datumaro/plugins/lfw_format.py @@ -8,7 +8,7 @@ from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, DatasetItem, - Importer, Points, SourceExtractor) + Importer, Label, LabelCategories, Points, SourceExtractor) from datumaro.util.image import find_images @@ -16,8 +16,9 @@ class LfwPath: IMAGES_DIR = 'images' LANDMARKS_FILE = 'landmarks.txt' PAIRS_FILE = 'pairs.txt' + PEOPLE_FILE = 'people.txt' IMAGE_EXT = '.jpg' - PATTERN = re.compile(r'([\w]+)_([-\d]+)') + PATTERN = re.compile(r'([\w-]+)_([-\d]+)') class LfwExtractor(SourceExtractor): def __init__(self, path, subset=None): @@ -29,14 +30,29 @@ def __init__(self, path, subset=None): super().__init__(subset=subset) self._dataset_dir = osp.dirname(osp.dirname(path)) + + people_file = osp.join(osp.dirname(path), LfwPath.PEOPLE_FILE) + self._categories = self._load_categories(people_file) + self._items = list(self._load_items(path).values()) + def _load_categories(self, path): + label_cat = LabelCategories() + if osp.isfile(path): + with open(path, encoding='utf-8') as labels_file: + for line in labels_file: + objects = line.strip().split('\t') + if len(objects) == 2: + label_cat.add(objects[0]) + return { AnnotationType.label: label_cat } + def _load_items(self, path): items = {} + label_categories = self._categories.get(AnnotationType.label) images_dir = osp.join(self._dataset_dir, self._subset, LfwPath.IMAGES_DIR) if osp.isdir(images_dir): - images = { osp.splitext(osp.relpath(p, images_dir))[0]: p + images = { osp.splitext(osp.relpath(p, images_dir))[0].replace('\\', '/'): p for p in find_images(images_dir, recursive=True) } else: images = {} @@ -44,44 +60,71 @@ def _load_items(self, path): with open(path, encoding='utf-8') as f: for line in f: pair = line.strip().split('\t') - if len(pair) == 3: - if pair[0] == '-': - image1 = pair[1] - image2 = pair[2] - else: - image1 = self.get_image_name(pair[0], pair[1]) - image2 = self.get_image_name(pair[0], pair[2]) - if image1 not in items: - items[image1] = DatasetItem(id=image1, subset=self._subset, - image=images.get(image1), - attributes={'positive_pairs': [], 'negative_pairs': []}) - if image2 not in items: - items[image2] = DatasetItem(id=image2, subset=self._subset, - image=images.get(image2), - attributes={'positive_pairs': [], 'negative_pairs': []}) + if len(pair) == 1 and pair[0] != '': + annotations = [] + image = pair[0] + item_id = pair[0] + objects = item_id.split('/') + if 1 < len(objects): + label_name = objects[0] + label = label_categories.find(label_name)[0] + if label != None: + annotations.append(Label(label)) + item_id = item_id[len(label_name) + 1:] + if item_id not in items: + items[item_id] = DatasetItem(id=item_id, subset=self._subset, + image=images.get(image), annotations=annotations) + elif len(pair) == 3: + image1, id1 = self.get_image_name(pair[0], pair[1]) + image2, id2 = self.get_image_name(pair[0], pair[2]) + label = label_categories.find(pair[0])[0] + if label == None: + raise Exception("Line %s: people file doesn't " + "contain person %s " % (line, pair[0])) + if id1 not in items: + annotations = [] + annotations.append(Label(label)) + items[id1] = DatasetItem(id=id1, subset=self._subset, + image=images.get(image1), annotations=annotations) + if id2 not in items: + annotations = [] + annotations.append(Label(label)) + items[id2] = DatasetItem(id=id2, subset=self._subset, + image=images.get(image2), annotations=annotations) # pairs form a directed graph - items[image1].attributes['positive_pairs'].append(image2) + if not items[id1].annotations[0].attributes.get('positive_pairs'): + items[id1].annotations[0].attributes['positive_pairs'] = [] + items[id1].annotations[0].attributes['positive_pairs'].append(image2) + elif len(pair) == 4: - if pair[0] == '-': - image1 = pair[1] - else: - image1 = self.get_image_name(pair[0], pair[1]) + image1, id1 = self.get_image_name(pair[0], pair[1]) if pair[2] == '-': image2 = pair[3] + id2 = pair[3] else: - image2 = self.get_image_name(pair[2], pair[3]) - if image1 not in items: - items[image1] = DatasetItem(id=image1, subset=self._subset, - image=images.get(image1), - attributes={'positive_pairs': [], 'negative_pairs': []}) - if image2 not in items: - items[image2] = DatasetItem(id=image2, subset=self._subset, - image=images.get(image2), - attributes={'positive_pairs': [], 'negative_pairs': []}) + image2, id2 = self.get_image_name(pair[2], pair[3]) + if id1 not in items: + annotations = [] + label = label_categories.find(pair[0])[0] + if label == None: + raise Exception("Line %s: people file doesn't " + "contain person %s " % (line, pair[0])) + annotations.append(Label(label)) + items[id1] = DatasetItem(id=id1, subset=self._subset, + image=images.get(image1), annotations=annotations) + if id2 not in items: + annotations = [] + label = label_categories.find(pair[2])[0] + if label != None: + annotations.append(Label(label)) + items[id2] = DatasetItem(id=id2, subset=self._subset, + image=images.get(image2), annotations=annotations) # pairs form a directed graph - items[image1].attributes['negative_pairs'].append(image2) + if not items[id1].annotations[0].attributes.get('negative_pairs'): + items[id1].annotations[0].attributes['negative_pairs'] = [] + items[id1].annotations[0].attributes['negative_pairs'].append(image2) landmarks_file = osp.join(self._dataset_dir, self._subset, LfwPath.LANDMARKS_FILE) @@ -91,10 +134,15 @@ def _load_items(self, path): line = line.split('\t') item_id = osp.splitext(line[0])[0] + objects = item_id.split('/') + if 1 < len(objects): + label_name = objects[0] + label = label_categories.find(label_name)[0] + if label != None: + item_id = item_id[len(label_name) + 1:] if item_id not in items: items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=osp.join(images_dir, line[0]), - attributes={'positive_pairs': [], 'negative_pairs': []}) + image=osp.join(images_dir, line[0])) annotations = items[item_id].annotations annotations.append(Points([float(p) for p in line[1:]])) @@ -103,7 +151,15 @@ def _load_items(self, path): @staticmethod def get_image_name(person, image_id): - return '{}/{}_{:04d}'.format(person, person, int(image_id)) + image, item_id = '', '' + try: + image_id = int(image_id) + image = '{}/{}_{:04d}'.format(person, person, image_id) + item_id = '{}_{:04d}'.format(person, image_id) + except ValueError: + image = '{}/{}'.format(person, image_id) + item_id = image_id + return image, item_id class LfwImporter(Importer): @classmethod @@ -115,42 +171,90 @@ class LfwConverter(Converter): def apply(self): for subset_name, subset in self._extractor.subsets().items(): + label_categories = self._extractor.categories()[AnnotationType.label] + labels = {} + for label in label_categories: + f = label.name + labels[label.name] = 0 + positive_pairs = [] negative_pairs = [] + neutral_items = [] landmarks = [] + included_items = [] for item in subset: + anns = [ann for ann in item.annotations + if ann.type == AnnotationType.label] + label, label_name = None, None + if anns: + label = anns[0] + label_name = label_categories[anns[0].label].name + labels[label_name] += 1 + if self._save_images and item.has_image: - self._save_image(item, - subdir=osp.join(subset_name, LfwPath.IMAGES_DIR)) - - search = LfwPath.PATTERN.search(item.id) - if search: - person1, num1 = search.groups() - num1 = int(num1) - else: - person1 = '-' + subdir=osp.join(subset_name, LfwPath.IMAGES_DIR) + if label_name: + subdir=osp.join(subdir, label_name) + self._save_image(item, subdir=subdir) + + if label != None: + person1 = label_name num1 = item.id - if 'positive_pairs' in item.attributes: - for pair in item.attributes['positive_pairs']: - search = LfwPath.PATTERN.search(pair) - if search: - num2 = search.groups()[1] - num2 = int(num2) - else: - num2 = pair - positive_pairs.append('%s\t%s\t%s' % (person1, num1, num2)) - if 'negative_pairs' in item.attributes: - for pair in item.attributes['negative_pairs']: - search = LfwPath.PATTERN.search(pair) - if search: - person2, num2 = search.groups() - num2 = int(num2) - else: - person2 = '-' - num2 = pair - negative_pairs.append('%s\t%s\t%s\t%s' % \ - (person1, num1, person2, num2)) + if num1.startswith(person1): + num1 = int(num1.replace(person1, '')[1:]) + curr_item = person1 + '/' + str(num1) + + if 'positive_pairs' in label.attributes: + if curr_item not in included_items: + included_items.append(curr_item) + for pair in label.attributes['positive_pairs']: + search = LfwPath.PATTERN.search(pair) + if search: + num2 = search.groups()[1] + num2 = int(num2) + else: + num2 = pair + if num2.startswith(person1): + num2 = num2.replace(person1, '')[1:] + curr_item = person1 + '/' + str(num2) + if curr_item not in included_items: + included_items.append(curr_item) + positive_pairs.append('%s\t%s\t%s' % (person1, num1, num2)) + + if 'negative_pairs' in label.attributes: + if curr_item not in included_items: + included_items.append(curr_item) + for pair in label.attributes['negative_pairs']: + search = LfwPath.PATTERN.search(pair) + curr_item = '' + if search: + person2, num2 = search.groups() + num2 = int(num2) + curr_item += person2 + '/' + else: + person2 = '-' + num2 = pair + objects = pair.split('/') + if 1 < len(objects) and objects[0] in labels: + person2 = objects[0] + num2 = pair.replace(person2, '')[1:] + curr_item += person2 + '/' + curr_item += str(num2) + if curr_item not in included_items: + included_items.append(curr_item) + negative_pairs.append('%s\t%s\t%s\t%s' % \ + (person1, num1, person2, num2)) + + if 'positive_pairs' not in label.attributes and \ + 'negative_pairs' not in label.attributes and \ + curr_item not in included_items: + neutral_items.append('%s/%s' % (person1, item.id)) + included_items.append(curr_item) + + elif item.id not in included_items: + neutral_items.append(item.id) + included_items.append(item.id) item_landmarks = [p for p in item.annotations if p.type == AnnotationType.points] @@ -163,9 +267,17 @@ def apply(self): with open(pairs_file, 'w', encoding='utf-8') as f: f.writelines(['%s\n' % pair for pair in positive_pairs]) f.writelines(['%s\n' % pair for pair in negative_pairs]) + f.writelines(['%s\n' % item for item in neutral_items]) if landmarks: landmarks_file = osp.join(self._save_dir, subset_name, LfwPath.LANDMARKS_FILE) with open(landmarks_file, 'w', encoding='utf-8') as f: f.writelines(['%s\n' % landmark for landmark in landmarks]) + + if labels: + people_file = osp.join(self._save_dir, subset_name, + LfwPath.PEOPLE_FILE) + with open(people_file, 'w', encoding='utf-8') as f: + f.writelines(['%s\t%d\n' % (label, labels[label]) + for label in labels]) diff --git a/docs/user_manual.md b/docs/user_manual.md index 06585d36fd..40f2f061b0 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -128,7 +128,7 @@ List of supported formats: - Market-1501 (`person re-identification`) - [Format specification](https://www.aitribune.com/dataset/2018051063) - [Dataset example](../tests/assets/market1501_dataset) -- LFW (`person re-identification`, `landmarks`) +- LFW (`classification`, `person re-identification`, `landmarks`) - [Format specification](http://vis-www.cs.umass.edu/lfw/) - [Dataset example](../tests/assets/lfw_dataset) diff --git a/tests/assets/lfw_dataset/test/people.txt b/tests/assets/lfw_dataset/test/people.txt new file mode 100644 index 0000000000..015b83c6af --- /dev/null +++ b/tests/assets/lfw_dataset/test/people.txt @@ -0,0 +1,3 @@ +2 +name0 2 +name1 2 \ No newline at end of file diff --git a/tests/test_lfw_format.py b/tests/test_lfw_format.py index 3aa64365d1..2adaae52a4 100644 --- a/tests/test_lfw_format.py +++ b/tests/test_lfw_format.py @@ -3,7 +3,7 @@ import numpy as np from datumaro.components.dataset import Dataset -from datumaro.components.extractor import DatasetItem, Points +from datumaro.components.extractor import DatasetItem, Label, Points from datumaro.plugins.lfw_format import LfwConverter, LfwImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -12,35 +12,33 @@ class LfwFormatTest(TestCase): def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ - DatasetItem(id='name0/name0_0001', - subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': ['name0/name0_0002'], - 'negative_pairs': [] - } + DatasetItem(id='name0_0001', subset='test', + image=np.ones((2, 5, 3)), + annotations=[Label(0, attributes={ + 'positive_pairs': ['name0/name0_0002'] + })] ), - DatasetItem(id='name0/name0_0002', - subset='test', image=np.ones((2, 5, 3)), - attributes={ + DatasetItem(id='name0_0002', subset='test', + image=np.ones((2, 5, 3)), + annotations=[Label(0, attributes={ 'positive_pairs': ['name0/name0_0001'], 'negative_pairs': ['name1/name1_0001'] - } + })] ), - DatasetItem(id='name1/name1_0001', - subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': ['name1/name1_0002'], - 'negative_pairs': [] - } + DatasetItem(id='name1_0001', subset='test', + image=np.ones((2, 5, 3)), + annotations=[Label(1, attributes={ + 'positive_pairs': ['name1/name1_0002'] + })] ), - DatasetItem(id='name1/name1_0002', - subset='test', image=np.ones((2, 5, 3)), - attributes={ + DatasetItem(id='name1_0002', subset='test', + image=np.ones((2, 5, 3)), + annotations=[Label(1, attributes={ 'positive_pairs': ['name1/name1_0002'], 'negative_pairs': ['name0/name0_0001'] - } + })] ), - ]) + ], categories=['name0', 'name1']) with TestDir() as test_dir: LfwConverter.convert(source_dataset, test_dir, save_images=True) @@ -50,27 +48,23 @@ def test_can_save_and_load(self): def test_can_save_and_load_with_landmarks(self): source_dataset = Dataset.from_iterable([ - DatasetItem(id='name0/name0_0001', + DatasetItem(id='name0_0001', subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': ['name0/name0_0002'], - 'negative_pairs': [] - }, annotations=[ + Label(0, attributes={ + 'positive_pairs': ['name0/name0_0002'] + }), Points([0, 4, 3, 3, 2, 2, 1, 0, 3, 0]), ] ), - DatasetItem(id='name0/name0_0002', + DatasetItem(id='name0_0002', subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': [], - 'negative_pairs': [] - }, annotations=[ + Label(0), Points([0, 5, 3, 5, 2, 2, 1, 0, 3, 0]), ] ), - ]) + ], categories=['name0']) with TestDir() as test_dir: LfwConverter.convert(source_dataset, test_dir, save_images=True) @@ -80,21 +74,45 @@ def test_can_save_and_load_with_landmarks(self): def test_can_save_and_load_with_no_subsets(self): source_dataset = Dataset.from_iterable([ - DatasetItem(id='name0/name0_0001', + DatasetItem(id='name0_0001', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': ['name0/name0_0002'], - 'negative_pairs': [] - }, + annotations=[Label(0, attributes={ + 'positive_pairs': ['name0/name0_0002'] + })], ), - DatasetItem(id='name0/name0_0002', + DatasetItem(id='name0_0002', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': [], - 'negative_pairs': [] - }, + annotations=[Label(0)] ), - ]) + ], categories=['name0']) + + with TestDir() as test_dir: + LfwConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'lfw') + + compare_datasets(self, source_dataset, parsed_dataset) + + def test_can_save_and_load_with_no_format_names(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='a/1', + image=np.ones((2, 5, 3)), + annotations=[Label(0, attributes={ + 'positive_pairs': ['name0/b/2'], + 'negative_pairs': ['d/4'] + })], + ), + DatasetItem(id='b/2', + image=np.ones((2, 5, 3)), + annotations=[Label(0)] + ), + DatasetItem(id='c/3', + image=np.ones((2, 5, 3)), + annotations=[Label(1)] + ), + DatasetItem(id='d/4', + image=np.ones((2, 5, 3)), + ), + ], categories=['name0', 'name1']) with TestDir() as test_dir: LfwConverter.convert(source_dataset, test_dir, save_images=True) @@ -105,20 +123,15 @@ def test_can_save_and_load_with_no_subsets(self): def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', - image=np.ones((2, 5, 3)), - attributes = { - 'positive_pairs': [], - 'negative_pairs': [] - }, + image=np.ones((2, 5, 3)) ), - DatasetItem(id='name0/name0_0002', + DatasetItem(id='name0_0002', image=np.ones((2, 5, 3)), - attributes = { - 'positive_pairs': [], + annotations=[Label(0, attributes={ 'negative_pairs': ['кириллица с пробелом'] - }, + })] ), - ]) + ], categories=['name0']) with TestDir() as test_dir: LfwConverter.convert(dataset, test_dir, save_images=True) @@ -128,21 +141,13 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ - DatasetItem(id='name0/name0_0001', image=Image( - path='name0/name0_0001.JPEG', data=np.zeros((4, 3, 3))), - attributes={ - 'positive_pairs': [], - 'negative_pairs': [] - }, - ), - DatasetItem(id='name0/name0_0002', image=Image( - path='name0/name0_0002.bmp', data=np.zeros((3, 4, 3))), - attributes={ - 'positive_pairs': ['name0/name0_0001'], - 'negative_pairs': [] - }, + DatasetItem(id='a/1', image=Image( + path='a/1.JPEG', data=np.zeros((4, 3, 3))), ), - ]) + DatasetItem(id='b/c/d/2', image=Image( + path='b/c/d/2.bmp', data=np.zeros((3, 4, 3))), + ), + ], categories=[]) with TestDir() as test_dir: LfwConverter.convert(dataset, test_dir, save_images=True) @@ -158,38 +163,33 @@ def test_can_detect(self): def test_can_import(self): expected_dataset = Dataset.from_iterable([ - DatasetItem(id='name0/name0_0001', - subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': [], - 'negative_pairs': ['name1/name1_0001', - 'name1/name1_0002'] - }, + DatasetItem(id='name0_0001', subset='test', + image=np.ones((2, 5, 3)), annotations=[ + Label(0, attributes={ + 'negative_pairs': ['name1/name1_0001', + 'name1/name1_0002'] + }), Points([0, 4, 3, 3, 2, 2, 1, 0, 3, 0]), ] ), - DatasetItem(id='name1/name1_0001', - subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': ['name1/name1_0002'], - 'negative_pairs': [] - }, + DatasetItem(id='name1_0001', subset='test', + image=np.ones((2, 5, 3)), annotations=[ + Label(1, attributes={ + 'positive_pairs': ['name1/name1_0002'], + }), Points([1, 6, 4, 6, 3, 3, 2, 1, 4, 1]), ] ), - DatasetItem(id='name1/name1_0002', - subset='test', image=np.ones((2, 5, 3)), - attributes={ - 'positive_pairs': [], - 'negative_pairs': [] - }, + DatasetItem(id='name1_0002', subset='test', + image=np.ones((2, 5, 3)), annotations=[ + Label(1), Points([0, 5, 3, 5, 2, 2, 1, 0, 3, 0]), ] ), - ]) + ], categories=['name0', 'name1']) dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'lfw') From 00d167cb04efdf423c68b95405d2087a19bcef5d Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Wed, 21 Apr 2021 16:15:16 +0900 Subject: [PATCH 12/40] Add splitter for segmentation task (#223) * added segmentation_split * updated changelog * rename reidentification to reid --- CHANGELOG.md | 1 + datumaro/plugins/splitter.py | 301 ++++++++++++----- docs/user_manual.md | 26 +- tests/test_splitter.py | 637 +++++++++++++++++++++++++++++------ 4 files changed, 763 insertions(+), 202 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4dffa8b25..a558f595a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Support for escaping in attribiute values in LabelMe format () +- Support for Segmentation Splitting () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index c9e19fa8b9..abc391ab19 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -5,6 +5,7 @@ import logging as log import numpy as np from math import gcd +from enum import Enum from datumaro.components.extractor import (Transform, AnnotationType, DEFAULT_SUBSET_NAME) @@ -13,34 +14,173 @@ NEAR_ZERO = 1e-7 +SplitTask = Enum( + "split", ["classification", "detection", "segmentation", "reid"] +) -class _TaskSpecificSplit(Transform, CliPlugin): - _default_split = [('train', 0.5), ('val', 0.2), ('test', 0.3)] + +class Split(Transform, CliPlugin): + """ + - classification split |n + Splits dataset into subsets(train/val/test) in class-wise manner. |n + Splits dataset images in the specified ratio, keeping the initial class + distribution.|n + |n + - detection & segmentation split |n + Each image can have multiple object annotations - + (bbox, mask, polygon). Since an image shouldn't be included + in multiple subsets at the same time, and image annotations + shoudln't be split, in general, dataset annotations are unlikely + to be split exactly in the specified ratio. |n + This split tries to split dataset images as close as possible + to the specified ratio, keeping the initial class distribution.|n + |n + - reidentification split |n + In this task, the test set should consist of images of unseen + people or objects during the training phase. |n + This function splits a dataset in the following way:|n + 1. Splits the dataset into 'train + val' and 'test' sets|n + |s|sbased on person or object ID.|n + 2. Splits 'test' set into 'test-gallery' and 'test-query' sets|n + |s|sin class-wise manner.|n + 3. Splits the 'train + val' set into 'train' and 'val' sets|n + |s|sin the same way.|n + The final subsets would be + 'train', 'val', 'test-gallery' and 'test-query'. |n + |n + Notes:|n + - Each image is expected to have only one Annotation. Unlabeled or + multi-labeled images will be split into subsets randomly. |n + - If Labels also have attributes, also splits by attribute values.|n + - If there is not enough images in some class or attributes group, + the split ratio can't be guaranteed.|n + In reidentification task, |n + - Object ID can be described by Label, or by attribute (--attr parameter)|n + - The splits of the test set are controlled by '--query' parameter |n + |s|sGallery ratio would be 1.0 - query.|n + |n + Example:|n + |s|s%(prog)s -t classification --subset train:.5 --subset val:.2 --subset test:.3 |n + |s|s%(prog)s -t detection --subset train:.5 --subset val:.2 --subset test:.3 |n + |s|s%(prog)s -t segmentation --subset train:.5 --subset val:.2 --subset test:.3 |n + |s|s%(prog)s -t reid --subset train:.5 --subset val:.2 --subset test:.3 --query .5 |n + Example: use 'person_id' attribute for splitting|n + |s|s%(prog)s --attr person_id + """ + + _default_split = [("train", 0.5), ("val", 0.2), ("test", 0.3)] + _default_query_ratio = 0.5 @classmethod def build_cmdline_parser(cls, **kwargs): parser = super().build_cmdline_parser(**kwargs) - parser.add_argument('-s', '--subset', action='append', - type=cls._split_arg, dest='splits', + parser.add_argument( + "-t", + "--task", + default=SplitTask.classification.name, + choices=[t.name for t in SplitTask], + help="(one of {}; default: %(default)s)".format( + ", ".join(t.name for t in SplitTask) + ), + ) + parser.add_argument( + "-s", + "--subset", + action="append", + type=cls._split_arg, + dest="splits", help="Subsets in the form: ':' " - "(repeatable, default: %s)" % dict(cls._default_split)) - parser.add_argument('--seed', type=int, help="Random seed") + "(repeatable, default: %s)" % dict(cls._default_split), + ) + parser.add_argument( + "--query", + type=float, + default=None, + help="Query ratio in the test set (default: %.3f)" + % cls._default_query_ratio, + ) + parser.add_argument( + "--attr", + type=str, + dest="attr_for_id", + default=None, + help="Attribute name representing the ID (default: use label)", + ) + parser.add_argument("--seed", type=int, help="Random seed") return parser @staticmethod def _split_arg(s): - parts = s.split(':') + parts = s.split(":") if len(parts) != 2: import argparse + raise argparse.ArgumentTypeError() return (parts[0], float(parts[1])) - def __init__(self, dataset, splits, seed, restrict=False): + def __init__(self, dataset, task, splits, query=None, attr_for_id=None, seed=None): super().__init__(dataset) if splits is None: splits = self._default_split + self.task = task + self.splitter = self._get_splitter( + task, dataset, splits, seed, query, attr_for_id + ) + self._initialized = False + self._subsets = self.splitter._subsets + + @staticmethod + def _get_splitter(task, dataset, splits, seed, query, attr_for_id): + if task == SplitTask.classification.name: + splitter = _ClassificationSplit(dataset=dataset, splits=splits, seed=seed) + elif task in {SplitTask.detection.name, SplitTask.segmentation.name}: + splitter = _InstanceSpecificSplit( + dataset=dataset, splits=splits, seed=seed, task=task + ) + elif task == SplitTask.reid.name: + splitter = _ReidentificationSplit( + dataset=dataset, + splits=splits, + seed=seed, + query=query, + attr_for_id=attr_for_id, + ) + else: + raise Exception( + f"Unknown task '{task}', available " + f"splitter format: {[a.name for a in SplitTask]}" + ) + return splitter + + def __iter__(self): + # lazy splitting + if self._initialized is False: + self.splitter._split_dataset() + self._initialized = True + for i, item in enumerate(self._extractor): + yield self.wrap_item(item, subset=self.splitter._find_split(i)) + + def get_subset(self, name): + # lazy splitting + if self._initialized is False: + self.splitter._split_dataset() + self._initialized = True + return super().get_subset(name) + + def subsets(self): + # lazy splitting + if self._initialized is False: + self.splitter._split_dataset() + self._initialized = True + return super().subsets() + + +class _TaskSpecificSplit: + def __init__(self, dataset, splits, seed, restrict=False): + self._extractor = dataset + snames, sratio, subsets = self._validate_splits(splits, restrict) self._snames = snames @@ -67,8 +207,7 @@ def _get_uniq_annotations(dataset): unlabeled_or_multi = [] for idx, item in enumerate(dataset): - labels = [a for a in item.annotations - if a.type == AnnotationType.label] + labels = [a for a in item.annotations if a.type == AnnotationType.label] if len(labels) == 1: annotations.append(labels[0]) else: @@ -86,11 +225,16 @@ def _validate_splits(splits, restrict=False): # remove subset name restriction # https://github.com/openvinotoolkit/datumaro/issues/194 if restrict: - assert subset in valid, \ - "Subset name must be one of %s, got %s" % (valid, subset) - assert 0.0 <= ratio and ratio <= 1.0, \ - "Ratio is expected to be in the range " \ - "[0, 1], but got %s for %s" % (ratio, subset) + assert subset in valid, "Subset name must be one of %s, got %s" % ( + valid, + subset, + ) + assert ( + 0.0 <= ratio and ratio <= 1.0 + ), "Ratio is expected to be in the range " "[0, 1], but got %s for %s" % ( + ratio, + subset, + ) # ignore near_zero ratio because it may produce partition error. if ratio > NEAR_ZERO: # handling duplication @@ -185,9 +329,9 @@ def _is_float(value): return by_attributes - def _split_by_attr(self, datasets, snames, ratio, out_splits, - merge_small_classes=True): - + def _split_by_attr( + self, datasets, snames, ratio, out_splits, merge_small_classes=True + ): def _split_indice(indice): sections, _ = self._get_sections(len(indice), ratio) splits = np.array_split(indice, sections) @@ -254,16 +398,8 @@ def _find_split(self, index): def _split_dataset(self): raise NotImplementedError() - def __iter__(self): - # lazy splitting - if self._initialized is False: - self._split_dataset() - self._initialized = True - for i, item in enumerate(self._extractor): - yield self.wrap_item(item, subset=self._find_split(i)) - -class ClassificationSplit(_TaskSpecificSplit): +class _ClassificationSplit(_TaskSpecificSplit): """ Splits dataset into subsets(train/val/test) in class-wise manner. |n Splits dataset images in the specified ratio, keeping the initial class @@ -277,8 +413,9 @@ class ClassificationSplit(_TaskSpecificSplit): the split ratio can't be guaranteed.|n |n Example:|n - |s|s%(prog)s --subset train:.5 --subset val:.2 --subset test:.3 + |s|s%(prog)s -t classification --subset train:.5 --subset val:.2 --subset test:.3 """ + def __init__(self, dataset, splits, seed=None): """ Parameters @@ -300,7 +437,7 @@ def _split_dataset(self): annotations, unlabeled = self._get_uniq_annotations(self._extractor) for idx, ann in enumerate(annotations): - label = getattr(ann, 'label', None) + label = getattr(ann, "label", None) if label not in by_labels: by_labels[label] = [] by_labels[label].append((idx, ann)) @@ -320,7 +457,7 @@ def _split_dataset(self): self._set_parts(by_splits) -class ReidentificationSplit(_TaskSpecificSplit): +class _ReidentificationSplit(_TaskSpecificSplit): """ Splits a dataset for re-identification task.|n Produces a split with a specified ratio of images, avoiding having same @@ -347,25 +484,14 @@ class ReidentificationSplit(_TaskSpecificSplit): |n Example: split a dataset in the specified ratio, split the test set|n |s|s|s|sinto gallery and query in 1:1 ratio|n - |s|s%(prog)s --subset train:.5 --subset val:.2 --subset test:.3 --query .5|n + |s|s%(prog)s -t reidentification --subset train:.5 --subset val:.2 --subset test:.3 --query .5|n Example: use 'person_id' attribute for splitting|n |s|s%(prog)s --attr person_id """ _default_query_ratio = 0.5 - @classmethod - def build_cmdline_parser(cls, **kwargs): - parser = super().build_cmdline_parser(**kwargs) - parser.add_argument('--query', type=float, - help="Query ratio in the test set (default: %.3f)" - % cls._default_query_ratio) - parser.add_argument('--attr', type=str, dest='attr_for_id', - help="Attribute name representing the ID (default: use label)") - return parser - - def __init__(self, dataset, splits, query=None, - attr_for_id=None, seed=None): + def __init__(self, dataset, splits, query=None, attr_for_id=None, seed=None): """ Parameters ---------- @@ -387,10 +513,10 @@ def __init__(self, dataset, splits, query=None, if query is None: query = self._default_query_ratio - assert 0.0 <= query and query <= 1.0, \ - "Query ratio is expected to be in the range " \ - "[0, 1], but got %f" % query - test_splits = [('test-query', query), ('test-gallery', 1.0 - query)] + assert 0.0 <= query and query <= 1.0, ( + "Query ratio is expected to be in the range " "[0, 1], but got %f" % query + ) + test_splits = [("test-query", query), ("test-gallery", 1.0 - query)] # remove subset name restriction self._subsets = {"train", "val", "test-gallery", "test-query"} @@ -410,15 +536,16 @@ def _split_dataset(self): annotations, unlabeled = self._get_uniq_annotations(dataset) if attr_for_id is None: # use label for idx, ann in enumerate(annotations): - ID = getattr(ann, 'label', None) + ID = getattr(ann, "label", None) if ID not in by_id: by_id[ID] = [] by_id[ID].append((idx, ann)) else: # use attr_for_id for idx, ann in enumerate(annotations): attributes = dict(ann.attributes.items()) - assert attr_for_id in attributes, \ + assert attr_for_id in attributes, ( "'%s' is expected as an attribute name" % attr_for_id + ) ID = attributes[attr_for_id] if ID not in by_id: by_id[ID] = [] @@ -426,9 +553,9 @@ def _split_dataset(self): required = self._get_required(id_ratio) if len(by_id) < required: - log.warning("There's not enough IDs, which is %s, " - "so train/val/test ratio can't be guaranteed." - % len(by_id) + log.warning( + "There's not enough IDs, which is %s, " + "so train/val/test ratio can't be guaranteed." % len(by_id) ) # 1. split dataset into trval and test @@ -444,7 +571,9 @@ def _split_dataset(self): trval = {pid: by_id[pid] for pid in splits[1]} # follow the ratio of datasetitems as possible. # naive heuristic: exchange the best item one by one. - expected_count = int(len(self._extractor) * split_ratio[0]) + expected_count = int( + (len(self._extractor) - len(unlabeled)) * split_ratio[0] + ) testset_total = int(np.sum([len(v) for v in testset.values()])) self._rebalancing(testset, trval, expected_count, testset_total) else: @@ -463,8 +592,9 @@ def _split_dataset(self): test_snames.append(sname) test_ratio.append(float(ratio)) - self._split_by_attr(testset, test_snames, test_ratio, by_splits, - merge_small_classes=False) + self._split_by_attr( + testset, test_snames, test_ratio, by_splits, merge_small_classes=False + ) # 3. split 'trval' into 'train' and 'val' trval_snames = ["train", "val"] @@ -479,14 +609,15 @@ def _split_dataset(self): total_ratio = np.sum(trval_ratio) if total_ratio < NEAR_ZERO: trval_splits = list(zip(["train", "val"], trval_ratio)) - log.warning("Sum of ratios is expected to be positive, " - "got %s, which is %s" - % (trval_splits, total_ratio) + log.warning( + "Sum of ratios is expected to be positive, " + "got %s, which is %s" % (trval_splits, total_ratio) ) else: trval_ratio /= total_ratio # normalize - self._split_by_attr(trval, trval_snames, trval_ratio, by_splits, - merge_small_classes=False) + self._split_by_attr( + trval, trval_snames, trval_ratio, by_splits, merge_small_classes=False + ) # split unlabeled data into 'not-supported'. if len(unlabeled) > 0: @@ -541,30 +672,16 @@ def _rebalancing(test, trval, expected_count, testset_total): test[id_trval] = trval.pop(id_trval) trval[id_test] = test.pop(id_test) - def get_subset(self, name): - # lazy splitting - if self._initialized is False: - self._split_dataset() - self._initialized = True - return super().get_subset(name) - - def subsets(self): - # lazy splitting - if self._initialized is False: - self._split_dataset() - self._initialized = True - return super().subsets() - -class DetectionSplit(_TaskSpecificSplit): +class _InstanceSpecificSplit(_TaskSpecificSplit): """ - Splits a dataset into subsets(train/val/test) for detection task, + Splits a dataset into subsets(train/val/test), using object annotations as a basis for splitting.|n Tries to produce an image split with the specified ratio, keeping the initial distribution of class objects.|n |n - In a detection dataset, each image can have multiple object annotations - - instance bounding boxes. Since an image shouldn't be included + each image can have multiple object annotations - + (instance bounding boxes, masks, polygons). Since an image shouldn't be included in multiple subsets at the same time, and image annotations shoudln't be split, in general, dataset annotations are unlikely to be split exactly in the specified ratio. |n @@ -572,14 +689,17 @@ class DetectionSplit(_TaskSpecificSplit): to the specified ratio, keeping the initial class distribution.|n |n Notes:|n - - Each image is expected to have one or more Bbox annotations.|n - - Only Bbox annotations are considered.|n + - Each image is expected to have one or more annotations.|n + - Only bbox annotations are considered in detection task.|n + - Mask or Polygon annotations are considered in segmentation task.|n |n Example: split dataset so that each object class annotations were split|n |s|s|s|sin the specified ratio between subsets|n - |s|s%(prog)s --subset train:.5 --subset val:.2 --subset test:.3 + |s|s%(prog)s -t detection --subset train:.5 --subset val:.2 --subset test:.3 |n + |s|s%(prog)s -t segmentation --subset train:.5 --subset val:.2 --subset test:.3 """ - def __init__(self, dataset, splits, seed=None): + + def __init__(self, dataset, splits, task, seed=None): """ Parameters ---------- @@ -591,18 +711,21 @@ def __init__(self, dataset, splits, seed=None): """ super().__init__(dataset, splits, seed) - @staticmethod - def _group_by_bbox_labels(dataset): + if task == SplitTask.detection.name: + self.annotation_type = [AnnotationType.bbox] + elif task == SplitTask.segmentation.name: + self.annotation_type = [AnnotationType.mask, AnnotationType.polygon] + + def _group_by_labels(self, dataset): by_labels = dict() unlabeled = [] for idx, item in enumerate(dataset): - bbox_anns = [a for a in item.annotations - if a.type == AnnotationType.bbox] + bbox_anns = [a for a in item.annotations if a.type in self.annotation_type] if len(bbox_anns) == 0: unlabeled.append(idx) continue for ann in bbox_anns: - label = getattr(ann, 'label', None) + label = getattr(ann, "label", None) if label not in by_labels: by_labels[label] = [(idx, ann)] else: @@ -615,7 +738,7 @@ def _split_dataset(self): subsets, sratio = self._snames, self._sratio # 1. group by bbox label - by_labels, unlabeled = self._group_by_bbox_labels(self._extractor) + by_labels, unlabeled = self._group_by_labels(self._extractor) # 2. group by attributes required = self._get_required(sratio) @@ -672,7 +795,7 @@ def _split_dataset(self): target_size = dict() expected = [] # expected numbers of per split GT samples for sname, ratio in zip(subsets, sratio): - target_size[sname] = total * ratio + target_size[sname] = (total - len(unlabeled)) * ratio expected.append([sname, np.array(n_combs) * ratio]) # functions for keep the # of annotations not exceed the expected num diff --git a/docs/user_manual.md b/docs/user_manual.md index 40f2f061b0..17f9afeaca 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1035,17 +1035,21 @@ datum transform -t random_split -- --subset train:.67 --subset test:.33 ``` Example: split a dataset in task-specific manner. Supported tasks are -classification, detection, and re-identification. +classification, detection, re-identification and segmentation. ``` bash -datum transform -t classification_split -- \ - --subset train:.5 --subset val:.2 --subset test:.3 +datum transform -t split -- \ + -t classification --subset train:.5 --subset val:.2 --subset test:.3 -datum transform -t detection_split -- \ - --subset train:.5 --subset val:.2 --subset test:.3 +datum transform -t split -- \ + -t detection --subset train:.5 --subset val:.2 --subset test:.3 -datum transform -t reidentification_split -- \ - --subset train:.5 --subset val:.2 --subset test:.3 --query .5 +datum transform -t split -- \ + -t segmentation --subset train:.5 --subset val:.2 --subset test:.3 + +datum transform -t split -- \ + -t reid --subset train:.5 --subset val:.2 --subset test:.3 \ + --query .5 ``` Example: convert polygons to masks, masks to boxes etc.: @@ -1076,7 +1080,7 @@ datum transform -t rename -- -e '|frame_(\d+)|\\1|' Example: Sampling dataset items, subset `train` is divided into `sampled`(sampled_subset) and `unsampled` - `train` has 100 data, and 20 samples are selected. There are `sampled`(20 samples) and 80 `unsampled`(80 datas) subsets. -- Remove `train` subset (if sample_name=`train` or unsample_name=`train`, still remain) +- Remove `train` subset (if sampled_subset=`train` or unsampled_name=`train`, still remain) - There are five methods of sampling the m option. - `topk`: Return the k with high uncertainty data - `lowk`: Return the k with low uncertainty data @@ -1087,9 +1091,9 @@ Example: Sampling dataset items, subset `train` is divided into `sampled`(sample ``` bash datum transform -t sampler -- \ -a entropy \ - -subset_name train \ - -sample_name sampled \ - -unsample_name unsampled \ + -i train \ + -o sampled \ + -u unsampled \ -m topk \ -k 20 ``` diff --git a/tests/test_splitter.py b/tests/test_splitter.py index 351162b9c9..4c233f0eb2 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -3,8 +3,15 @@ from unittest import TestCase from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, Label, Bbox, - LabelCategories, AnnotationType) +from datumaro.components.extractor import ( + DatasetItem, + Label, + Bbox, + Mask, + Polygon, + LabelCategories, + AnnotationType, +) import datumaro.plugins.splitter as splitter from datumaro.components.operations import compute_ann_statistics @@ -40,20 +47,23 @@ def _generate_dataset(self, config): for _ in range(count): idx += 1 iterable.append( - DatasetItem(idx, subset=self._get_subset(idx), - annotations=[ - Label(label_id, attributes=attributes) - ], - image=np.ones((1, 1, 3)) + DatasetItem( + idx, + subset=self._get_subset(idx), + annotations=[Label(label_id, attributes=attributes)], + image=np.ones((1, 1, 3)), ) ) else: for _ in range(counts): idx += 1 iterable.append( - DatasetItem(idx, subset=self._get_subset(idx), + DatasetItem( + idx, + subset=self._get_subset(idx), annotations=[Label(label_id)], - image=np.ones((1, 1, 3))) + image=np.ones((1, 1, 3)), + ) ) categories = {AnnotationType.label: label_cat} dataset = Dataset.from_iterable(iterable, categories) @@ -66,9 +76,10 @@ def test_split_for_classification_multi_class_no_attr(self): "label3": {"attrs": None, "counts": 30}, } source = self._generate_dataset(config) + task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(42, len(actual.get_subset("train"))) self.assertEqual(18, len(actual.get_subset("test"))) @@ -91,9 +102,10 @@ def test_split_for_classification_single_class_single_attr(self): counts = {0: 10, 1: 20, 2: 30} config = {"label": {"attrs": ["attr"], "counts": counts}} source = self._generate_dataset(config) + task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(42, len(actual.get_subset("train"))) self.assertEqual(18, len(actual.get_subset("test"))) @@ -124,10 +136,11 @@ def test_split_for_classification_single_class_multi_attr(self): attrs = ["attr1", "attr2"] config = {"label": {"attrs": attrs, "counts": counts}} source = self._generate_dataset(config) + task = splitter.SplitTask.classification.name with self.subTest("zero remainder"): splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(84, len(actual.get_subset("train"))) self.assertEqual(36, len(actual.get_subset("test"))) @@ -152,7 +165,7 @@ def test_split_for_classification_single_class_multi_attr(self): with self.subTest("non-zero remainder"): splits = [("train", 0.95), ("test", 0.05)] - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(114, len(actual.get_subset("train"))) self.assertEqual(6, len(actual.get_subset("test"))) @@ -173,9 +186,10 @@ def test_split_for_classification_multi_label_with_attr(self): "label2": {"attrs": attr2, "counts": counts}, } source = self._generate_dataset(config) + task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) train = actual.get_subset("train") test = actual.get_subset("test") @@ -213,12 +227,10 @@ def test_split_for_classification_multi_label_with_attr(self): self.assertEqual(15, attr_test["attr3"]["distribution"]["2"][0]) with self.subTest("random seed test"): - r1 = splitter.ClassificationSplit(source, splits, seed=1234) - r2 = splitter.ClassificationSplit(source, splits, seed=1234) - r3 = splitter.ClassificationSplit(source, splits, seed=4321) - self.assertEqual( - list(r1.get_subset("test")), list(r2.get_subset("test")) - ) + r1 = splitter.Split(source, task, splits, seed=1234) + r2 = splitter.Split(source, task, splits, seed=1234) + r3 = splitter.Split(source, task, splits, seed=4321) + self.assertEqual(list(r1.get_subset("test")), list(r2.get_subset("test"))) self.assertNotEqual( list(r1.get_subset("test")), list(r3.get_subset("test")) ) @@ -229,8 +241,9 @@ def test_split_for_classification_zero_ratio(self): } source = self._generate_dataset(config) splits = [("train", 0.1), ("val", 0.9), ("test", 0.0)] + task = splitter.SplitTask.classification.name - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(1, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) @@ -241,7 +254,8 @@ def test_split_for_classification_unlabeled(self): iterable = [DatasetItem(i, annotations=[]) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + task = splitter.SplitTask.classification.name + actual = splitter.Split(source, task, splits) self.assertEqual(7, len(actual.get_subset("train"))) self.assertEqual(3, len(actual.get_subset("test"))) @@ -251,35 +265,41 @@ def test_split_for_classification_unlabeled(self): iterable = [DatasetItem(i, annotations=anns) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.ClassificationSplit(source, splits) + task = splitter.SplitTask.classification.name + actual = splitter.Split(source, task, splits) self.assertEqual(7, len(actual.get_subset("train"))) self.assertEqual(3, len(actual.get_subset("test"))) def test_split_for_classification_gives_error(self): - source = Dataset.from_iterable([ - DatasetItem(1, annotations=[Label(0)]), - DatasetItem(2, annotations=[Label(1)]), - ], categories=["a", "b", "c"]) + source = Dataset.from_iterable( + [ + DatasetItem(1, annotations=[Label(0)]), + DatasetItem(2, annotations=[Label(1)]), + ], + categories=["a", "b", "c"], + ) + task = splitter.SplitTask.classification.name with self.subTest("wrong ratio"): with self.assertRaisesRegex(Exception, "in the range"): splits = [("train", -0.5), ("test", 1.5)] - splitter.ClassificationSplit(source, splits) + splitter.Split(source, task, splits) with self.assertRaisesRegex(Exception, "Sum of ratios"): splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] - splitter.ClassificationSplit(source, splits) + splitter.Split(source, task, splits) with self.subTest("duplicated subset name"): with self.assertRaisesRegex(Exception, "duplicated"): splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] - splitter.ClassificationSplit(source, splits) + splitter.Split(source, task, splits) def test_split_for_reidentification(self): - ''' + """ Test ReidentificationSplit using Dataset with label (ImageNet style) - ''' + """ + def _get_present(stat): values_present = [] for label, dist in stat["distribution"].items(): @@ -303,9 +323,9 @@ def _get_present(stat): attr_for_id = None source = self._generate_dataset(config) splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] + task = splitter.SplitTask.reid.name query = 0.4 / 0.7 - actual = splitter.ReidentificationSplit(source, - splits, query, attr_for_id) + actual = splitter.Split(source, task, splits, query, attr_for_id) stats = dict() for sname in ["train", "val", "test-query", "test-gallery"]: @@ -353,9 +373,9 @@ def _get_present(stat): self.assertEqual(int(total * 0.4 / 0.7), dist_query[pid][0]) def test_split_for_reidentification_randomseed(self): - ''' + """ Test randomseed for reidentification - ''' + """ counts = {} config = dict() for i in range(10): @@ -364,30 +384,28 @@ def test_split_for_reidentification_randomseed(self): counts[label] = count config[label] = {"attrs": None, "counts": count} source = self._generate_dataset(config) + task = splitter.SplitTask.reid.name splits = [("train", 0.5), ("test", 0.5)] query = 0.4 / 0.7 - r1 = splitter.ReidentificationSplit(source, splits, query, seed=1234) - r2 = splitter.ReidentificationSplit(source, splits, query, seed=1234) - r3 = splitter.ReidentificationSplit(source, splits, query, seed=4321) - self.assertEqual( - list(r1.get_subset("train")), list(r2.get_subset("train")) - ) - self.assertNotEqual( - list(r1.get_subset("train")), list(r3.get_subset("train")) - ) + r1 = splitter.Split(source, task, splits, query, seed=1234) + r2 = splitter.Split(source, task, splits, query, seed=1234) + r3 = splitter.Split(source, task, splits, query, seed=4321) + self.assertEqual(list(r1.get_subset("train")), list(r2.get_subset("train"))) + self.assertNotEqual(list(r1.get_subset("train")), list(r3.get_subset("train"))) def test_split_for_reidentification_rebalance(self): - ''' + """ rebalance function shouldn't gives error when there's no exchange - ''' + """ config = dict() for i in range(100): label = "label%03d" % i config[label] = {"attrs": None, "counts": 7} source = self._generate_dataset(config) + task = splitter.SplitTask.reid.name splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] query = 0.4 / 0.7 - actual = splitter.ReidentificationSplit(source, splits, query) + actual = splitter.Split(source, task, splits, query) self.assertEqual(350, len(actual.get_subset("train"))) self.assertEqual(140, len(actual.get_subset("val"))) @@ -396,12 +414,13 @@ def test_split_for_reidentification_rebalance(self): def test_split_for_reidentification_unlabeled(self): query = 0.5 + task = splitter.SplitTask.reid.name with self.subTest("no label"): iterable = [DatasetItem(i, annotations=[]) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.6), ("test", 0.4)] - actual = splitter.ReidentificationSplit(source, splits, query) + actual = splitter.Split(source, task, splits, query) self.assertEqual(10, len(actual.get_subset("not-supported"))) with self.subTest("multi label"): @@ -409,12 +428,13 @@ def test_split_for_reidentification_unlabeled(self): iterable = [DatasetItem(i, annotations=anns) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.6), ("test", 0.4)] - actual = splitter.ReidentificationSplit(source, splits, query) + actual = splitter.Split(source, task, splits, query) self.assertEqual(10, len(actual.get_subset("not-supported"))) def test_split_for_reidentification_gives_error(self): query = 0.4 / 0.7 # valid query ratio + task = splitter.SplitTask.reid.name counts = {i: (i % 3 + 1) * 7 for i in range(10)} config = {"person": {"attrs": ["PID"], "counts": counts}} @@ -422,35 +442,35 @@ def test_split_for_reidentification_gives_error(self): with self.subTest("wrong ratio"): with self.assertRaisesRegex(Exception, "in the range"): splits = [("train", -0.5), ("val", 0.2), ("test", 0.3)] - splitter.ReidentificationSplit(source, splits, query) + splitter.Split(source, task, splits, query) with self.assertRaisesRegex(Exception, "Sum of ratios"): splits = [("train", 0.6), ("val", 0.2), ("test", 0.3)] - splitter.ReidentificationSplit(source, splits, query) + splitter.Split(source, task, splits, query) with self.assertRaisesRegex(Exception, "in the range"): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.ReidentificationSplit(source, splits, -query) + actual = splitter.Split(source, task, splits, -query) with self.subTest("duplicated subset name"): with self.assertRaisesRegex(Exception, "duplicated"): splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] - splitter.ReidentificationSplit(source, splits, query) + splitter.Split(source, task, splits, query) with self.subTest("wrong subset name"): with self.assertRaisesRegex(Exception, "Subset name"): splits = [("_train", 0.5), ("val", 0.2), ("test", 0.3)] - splitter.ReidentificationSplit(source, splits, query) + splitter.Split(source, task, splits, query) with self.subTest("wrong attribute name for person id"): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.ReidentificationSplit(source, splits, query) + actual = splitter.Split(source, task, splits, query) with self.assertRaisesRegex(Exception, "Unknown subset"): actual.get_subset("test") - def _generate_detection_dataset(self, **kwargs): - append_bbox = kwargs.get("append_bbox") + def _generate_detection_segmentation_dataset(self, **kwargs): + annotation_type = kwargs.get("annotation_type") with_attr = kwargs.get("with_attr", False) nimages = kwargs.get("nimages", 10) @@ -479,10 +499,18 @@ def _generate_detection_dataset(self, **kwargs): attributes["attr0"] = attr_val % 3 attributes["attr%d" % (label_id + 1)] = attr_val % 2 for ann_id in range(count): - append_bbox(annotations, label_id=label_id, ann_id=ann_id, - attributes=attributes) - item = DatasetItem(img_id, subset=self._get_subset(img_id), - annotations=annotations, attributes={"id": img_id}) + annotation_type( + annotations, + label_id=label_id, + ann_id=ann_id, + attributes=attributes, + ) + item = DatasetItem( + img_id, + subset=self._get_subset(img_id), + annotations=annotations, + attributes={"id": img_id}, + ) iterable.append(item) dataset = Dataset.from_iterable(iterable, categories) @@ -492,7 +520,12 @@ def _generate_detection_dataset(self, **kwargs): def _get_append_bbox(dataset_type): def append_bbox_coco(annotations, **kwargs): annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"], + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"], id=kwargs["ann_id"], attributes=kwargs["attributes"], group=kwargs["ann_id"], @@ -504,7 +537,12 @@ def append_bbox_coco(annotations, **kwargs): def append_bbox_voc(annotations, **kwargs): annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"], + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"], id=kwargs["ann_id"] + 1, attributes=kwargs["attributes"], group=kwargs["ann_id"], @@ -514,7 +552,12 @@ def append_bbox_voc(annotations, **kwargs): Label(kwargs["label_id"], attributes=kwargs["attributes"]) ) annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"] + 3, + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"] + 3, group=kwargs["ann_id"], ) ) # part @@ -530,7 +573,12 @@ def append_bbox_yolo(annotations, **kwargs): def append_bbox_cvat(annotations, **kwargs): annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"], + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"], id=kwargs["ann_id"], attributes=kwargs["attributes"], group=kwargs["ann_id"], @@ -543,7 +591,12 @@ def append_bbox_cvat(annotations, **kwargs): def append_bbox_labelme(annotations, **kwargs): annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"], + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"], id=kwargs["ann_id"], attributes=kwargs["attributes"], ) @@ -554,7 +607,12 @@ def append_bbox_labelme(annotations, **kwargs): def append_bbox_mot(annotations, **kwargs): annotations.append( - Bbox(1, 1, 2, 2, label=kwargs["label_id"], + Bbox( + 1, + 1, + 2, + 2, + label=kwargs["label_id"], attributes=kwargs["attributes"], ) ) @@ -563,9 +621,7 @@ def append_bbox_mot(annotations, **kwargs): ) def append_bbox_widerface(annotations, **kwargs): - annotations.append( - Bbox(1, 1, 2, 2, attributes=kwargs["attributes"]) - ) + annotations.append(Bbox(1, 1, 2, 2, attributes=kwargs["attributes"])) annotations.append(Label(0, attributes=kwargs["attributes"])) functions = { @@ -581,8 +637,169 @@ def append_bbox_widerface(annotations, **kwargs): func = functions.get(dataset_type, append_bbox_cvat) return func + @staticmethod + def _get_append_mask(dataset_type): + def append_mask_coco(annotations, **kwargs): + annotations.append( + Mask( + np.array([[0, 0, 0, 1, 0]]), + label=kwargs["label_id"], + id=kwargs["ann_id"], + attributes=kwargs["attributes"], + group=kwargs["ann_id"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + def append_mask_voc(annotations, **kwargs): + annotations.append( + Mask( + np.array([[0, 0, 0, 1, 0]]), + label=kwargs["label_id"], + id=kwargs["ann_id"] + 1, + attributes=kwargs["attributes"], + group=kwargs["ann_id"], + ) + ) # obj + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + annotations.append( + Mask( + np.array([[0, 0, 0, 1, 0]]), + label=kwargs["label_id"] + 3, + group=kwargs["ann_id"], + ) + ) # part + annotations.append( + Label(kwargs["label_id"] + 3, attributes=kwargs["attributes"]) + ) + + def append_mask_labelme(annotations, **kwargs): + annotations.append( + Mask( + np.array([[0, 0, 0, 1, 0]]), + label=kwargs["label_id"], + id=kwargs["ann_id"], + attributes=kwargs["attributes"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + def append_mask_mot(annotations, **kwargs): + annotations.append( + Mask( + np.array([[0, 0, 0, 1, 0]]), + label=kwargs["label_id"], + attributes=kwargs["attributes"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + functions = { + "coco": append_mask_coco, + "voc": append_mask_voc, + "labelme": append_mask_labelme, + "mot": append_mask_mot, + } + + func = functions.get(dataset_type, append_mask_coco) + return func + + @staticmethod + def _get_append_polygon(dataset_type): + def append_polygon_coco(annotations, **kwargs): + annotations.append( + Polygon( + [0, 0, 1, 0, 1, 2, 0, 2], + label=kwargs["label_id"], + id=kwargs["ann_id"], + attributes=kwargs["attributes"], + group=kwargs["ann_id"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + def append_polygon_voc(annotations, **kwargs): + annotations.append( + Polygon( + [0, 0, 1, 0, 1, 2, 0, 2], + label=kwargs["label_id"], + id=kwargs["ann_id"] + 1, + attributes=kwargs["attributes"], + group=kwargs["ann_id"], + ) + ) # obj + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + annotations.append( + Polygon( + [0, 0, 1, 0, 1, 2, 0, 2], + label=kwargs["label_id"] + 3, + group=kwargs["ann_id"], + ) + ) # part + annotations.append( + Label(kwargs["label_id"] + 3, attributes=kwargs["attributes"]) + ) + + def append_polygon_yolo(annotations, **kwargs): + annotations.append(Bbox(1, 1, 2, 2, label=kwargs["label_id"])) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + def append_polygon_cvat(annotations, **kwargs): + annotations.append( + Polygon( + [0, 0, 1, 0, 1, 2, 0, 2], + label=kwargs["label_id"], + id=kwargs["ann_id"], + attributes=kwargs["attributes"], + group=kwargs["ann_id"], + z_order=kwargs["ann_id"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + def append_polygon_labelme(annotations, **kwargs): + annotations.append( + Polygon( + [0, 0, 1, 0, 1, 2, 0, 2], + label=kwargs["label_id"], + id=kwargs["ann_id"], + attributes=kwargs["attributes"], + ) + ) + annotations.append( + Label(kwargs["label_id"], attributes=kwargs["attributes"]) + ) + + functions = { + "coco": append_polygon_coco, + "voc": append_polygon_voc, + "yolo": append_polygon_yolo, + "cvat": append_polygon_cvat, + "labelme": append_polygon_labelme, + } + + func = functions.get(dataset_type, append_polygon_coco) + return func + def test_split_for_detection(self): dtypes = ["coco", "voc", "yolo", "cvat", "labelme", "mot", "widerface"] + task = splitter.SplitTask.detection.name params = [] for dtype in dtypes: for with_attr in [False, True]: @@ -590,8 +807,8 @@ def test_split_for_detection(self): params.append((dtype, with_attr, 10, 7, 0, 3)) for dtype, with_attr, nimages, train, val, test in params: - source, _ = self._generate_detection_dataset( - append_bbox=self._get_append_bbox(dtype), + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_bbox(dtype), with_attr=with_attr, nimages=nimages, ) @@ -608,34 +825,31 @@ def test_split_for_detection(self): train=train, val=val, test=test, + task=task, ): - actual = splitter.DetectionSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(train, len(actual.get_subset("train"))) self.assertEqual(val, len(actual.get_subset("val"))) self.assertEqual(test, len(actual.get_subset("test"))) # random seed test - source, _ = self._generate_detection_dataset( - append_bbox=self._get_append_bbox("cvat"), + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_bbox("cvat"), with_attr=True, nimages=10, ) splits = [("train", 0.5), ("test", 0.5)] - r1 = splitter.DetectionSplit(source, splits, seed=1234) - r2 = splitter.DetectionSplit(source, splits, seed=1234) - r3 = splitter.DetectionSplit(source, splits, seed=4321) - self.assertEqual( - list(r1.get_subset("test")), list(r2.get_subset("test")) - ) - self.assertNotEqual( - list(r1.get_subset("test")), list(r3.get_subset("test")) - ) + r1 = splitter.Split(source, task, splits, seed=1234) + r2 = splitter.Split(source, task, splits, seed=1234) + r3 = splitter.Split(source, task, splits, seed=4321) + self.assertEqual(list(r1.get_subset("test")), list(r2.get_subset("test"))) + self.assertNotEqual(list(r1.get_subset("test")), list(r3.get_subset("test"))) def test_split_for_detection_with_unlabeled(self): - source, _ = self._generate_detection_dataset( - append_bbox=self._get_append_bbox("cvat"), + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_bbox("cvat"), with_attr=True, nimages=10, ) @@ -643,42 +857,48 @@ def test_split_for_detection_with_unlabeled(self): source.put(DatasetItem(i + 10, annotations={})) splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] - actual = splitter.DetectionSplit(source, splits) + task = splitter.SplitTask.detection.name + actual = splitter.Split(source, task, splits) self.assertEqual(10, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) def test_split_for_detection_gives_error(self): - source, _ = self._generate_detection_dataset( - append_bbox=self._get_append_bbox("cvat"), + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_bbox("cvat"), with_attr=True, nimages=5, ) + task = splitter.SplitTask.detection.name with self.subTest("wrong ratio"): with self.assertRaisesRegex(Exception, "in the range"): splits = [("train", -0.5), ("test", 1.5)] - splitter.DetectionSplit(source, splits) + splitter.Split(source, task, splits) with self.assertRaisesRegex(Exception, "Sum of ratios"): splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] - splitter.DetectionSplit(source, splits) + splitter.Split(source, task, splits) with self.subTest("duplicated subset name"): with self.assertRaisesRegex(Exception, "duplicated"): splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] - splitter.DetectionSplit(source, splits) + splitter.Split(source, task, splits) def test_no_subset_name_and_count_restriction(self): - splits = [("_train", 0.5), ("valid", 0.1), ("valid2", 0.1), - ("test*", 0.2), ("test2", 0.1)] + splits = [ + ("_train", 0.5), + ("valid", 0.1), + ("valid2", 0.1), + ("test*", 0.2), + ("test2", 0.1), + ] with self.subTest("classification"): - config = { - "label1": {"attrs": None, "counts": 10} - } + config = {"label1": {"attrs": None, "counts": 10}} + task = splitter.SplitTask.classification.name source = self._generate_dataset(config) - actual = splitter.ClassificationSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(5, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) self.assertEqual(1, len(actual.get_subset("valid2"))) @@ -686,14 +906,227 @@ def test_no_subset_name_and_count_restriction(self): self.assertEqual(1, len(actual.get_subset("test2"))) with self.subTest("detection"): - source, _ = self._generate_detection_dataset( - append_bbox=self._get_append_bbox("cvat"), + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_bbox("cvat"), + with_attr=True, + nimages=10, + ) + task = splitter.SplitTask.detection.name + actual = splitter.Split(source, task, splits) + self.assertEqual(5, len(actual.get_subset("_train"))) + self.assertEqual(1, len(actual.get_subset("valid"))) + self.assertEqual(1, len(actual.get_subset("valid2"))) + self.assertEqual(2, len(actual.get_subset("test*"))) + self.assertEqual(1, len(actual.get_subset("test2"))) + + with self.subTest("segmentation"): + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_mask("coco"), + with_attr=True, + nimages=10, + ) + task = splitter.SplitTask.detection.name + actual = splitter.Split(source, task, splits) + self.assertEqual(5, len(actual.get_subset("_train"))) + self.assertEqual(1, len(actual.get_subset("valid"))) + self.assertEqual(1, len(actual.get_subset("valid2"))) + self.assertEqual(2, len(actual.get_subset("test*"))) + self.assertEqual(1, len(actual.get_subset("test2"))) + + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_polygon("coco"), with_attr=True, nimages=10, ) - actual = splitter.DetectionSplit(source, splits) + actual = splitter.Split(source, task, splits) self.assertEqual(5, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) self.assertEqual(1, len(actual.get_subset("valid2"))) self.assertEqual(2, len(actual.get_subset("test*"))) self.assertEqual(1, len(actual.get_subset("test2"))) + + def test_split_for_segmentation(self): + + with self.subTest("mask annotation"): + dtypes = ["coco", "voc", "labelme", "mot"] + task = splitter.SplitTask.segmentation.name + params = [] + for dtype in dtypes: + for with_attr in [False, True]: + params.append((dtype, with_attr, 10, 5, 3, 2)) + params.append((dtype, with_attr, 10, 7, 0, 3)) + + for dtype, with_attr, nimages, train, val, test in params: + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_mask(dtype), + with_attr=with_attr, + nimages=nimages, + ) + total = np.sum([train, val, test]) + splits = [ + ("train", train / total), + ("val", val / total), + ("test", test / total), + ] + with self.subTest( + dtype=dtype, + with_attr=with_attr, + nimage=nimages, + train=train, + val=val, + test=test, + task=task, + ): + actual = splitter.Split(source, task, splits) + + self.assertEqual(train, len(actual.get_subset("train"))) + self.assertEqual(val, len(actual.get_subset("val"))) + self.assertEqual(test, len(actual.get_subset("test"))) + + # random seed test + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_mask("coco"), + with_attr=True, + nimages=10, + ) + + splits = [("train", 0.5), ("test", 0.5)] + r1 = splitter.Split(source, task, splits, seed=1234) + r2 = splitter.Split(source, task, splits, seed=1234) + r3 = splitter.Split(source, task, splits, seed=4321) + self.assertEqual(list(r1.get_subset("test")), list(r2.get_subset("test"))) + self.assertNotEqual( + list(r1.get_subset("test")), list(r3.get_subset("test")) + ) + + with self.subTest("polygon annotation"): + dtypes = ["coco", "voc", "labelme", "yolo", "cvat"] + task = splitter.SplitTask.segmentation.name + params = [] + for dtype in dtypes: + for with_attr in [False, True]: + params.append((dtype, with_attr, 10, 5, 3, 2)) + params.append((dtype, with_attr, 10, 7, 0, 3)) + + for dtype, with_attr, nimages, train, val, test in params: + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_polygon(dtype), + with_attr=with_attr, + nimages=nimages, + ) + total = np.sum([train, val, test]) + splits = [ + ("train", train / total), + ("val", val / total), + ("test", test / total), + ] + with self.subTest( + dtype=dtype, + with_attr=with_attr, + nimage=nimages, + train=train, + val=val, + test=test, + task=task, + ): + actual = splitter.Split(source, task, splits) + + self.assertEqual(train, len(actual.get_subset("train"))) + self.assertEqual(val, len(actual.get_subset("val"))) + self.assertEqual(test, len(actual.get_subset("test"))) + + # random seed test + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_polygon("coco"), + with_attr=True, + nimages=10, + ) + + splits = [("train", 0.5), ("test", 0.5)] + r1 = splitter.Split(source, task, splits, seed=1234) + r2 = splitter.Split(source, task, splits, seed=1234) + r3 = splitter.Split(source, task, splits, seed=4321) + self.assertEqual(list(r1.get_subset("test")), list(r2.get_subset("test"))) + self.assertNotEqual( + list(r1.get_subset("test")), list(r3.get_subset("test")) + ) + + def test_split_for_segmentation_with_unlabeled(self): + + with self.subTest("mask annotation"): + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_mask("coco"), + with_attr=True, + nimages=10, + ) + for i in range(10): + source.put(DatasetItem(i + 10, annotations={})) + + splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] + task = splitter.SplitTask.segmentation.name + actual = splitter.Split(source, task, splits) + self.assertEqual(10, len(actual.get_subset("train"))) + self.assertEqual(4, len(actual.get_subset("val"))) + self.assertEqual(6, len(actual.get_subset("test"))) + + with self.subTest("polygon annotation"): + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_polygon("coco"), + with_attr=True, + nimages=10, + ) + for i in range(10): + source.put(DatasetItem(i + 10, annotations={})) + + splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] + task = splitter.SplitTask.segmentation.name + actual = splitter.Split(source, task, splits) + self.assertEqual(10, len(actual.get_subset("train"))) + self.assertEqual(4, len(actual.get_subset("val"))) + self.assertEqual(6, len(actual.get_subset("test"))) + + def test_split_for_segmentation_gives_error(self): + + with self.subTest("mask annotation"): + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_mask("coco"), + with_attr=True, + nimages=5, + ) + task = splitter.SplitTask.segmentation.name + + with self.subTest("wrong ratio"): + with self.assertRaisesRegex(Exception, "in the range"): + splits = [("train", -0.5), ("test", 1.5)] + splitter.Split(source, task, splits) + + with self.assertRaisesRegex(Exception, "Sum of ratios"): + splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] + splitter.Split(source, task, splits) + + with self.subTest("duplicated subset name"): + with self.assertRaisesRegex(Exception, "duplicated"): + splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] + splitter.Split(source, task, splits) + + with self.subTest("polygon annotation"): + source, _ = self._generate_detection_segmentation_dataset( + annotation_type=self._get_append_polygon("coco"), + with_attr=True, + nimages=5, + ) + task = splitter.SplitTask.segmentation.name + + with self.subTest("wrong ratio"): + with self.assertRaisesRegex(Exception, "in the range"): + splits = [("train", -0.5), ("test", 1.5)] + splitter.Split(source, task, splits) + + with self.assertRaisesRegex(Exception, "Sum of ratios"): + splits = [("train", 0.5), ("test", 0.5), ("val", 0.5)] + splitter.Split(source, task, splits) + + with self.subTest("duplicated subset name"): + with self.assertRaisesRegex(Exception, "duplicated"): + splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] + splitter.Split(source, task, splits) From b9469d9aa709c6371f13ba1f1108cf4c43c0c7a4 Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Mon, 26 Apr 2021 14:56:25 +0300 Subject: [PATCH 13/40] Support for CIFAR-10/100 format (#225) * add CIFAR dataset format * add CIFAR to documentation * update Changelog --- CHANGELOG.md | 1 + README.md | 1 + datumaro/plugins/cifar_format.py | 181 ++++++++++++++++++++++++ docs/user_manual.md | 3 + tests/assets/cifar_dataset/batches.meta | Bin 0 -> 70 bytes tests/assets/cifar_dataset/data_batch_1 | Bin 0 -> 3317 bytes tests/assets/cifar_dataset/test_batch | Bin 0 -> 9494 bytes tests/test_cifar_format.py | 149 +++++++++++++++++++ 8 files changed, 335 insertions(+) create mode 100644 datumaro/plugins/cifar_format.py create mode 100644 tests/assets/cifar_dataset/batches.meta create mode 100644 tests/assets/cifar_dataset/data_batch_1 create mode 100644 tests/assets/cifar_dataset/test_batch create mode 100644 tests/test_cifar_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a558f595a6..ea1be7bcde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Support for escaping in attribiute values in LabelMe format () - Support for Segmentation Splitting () +- Support for CIFAR-10/100 dataset format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/README.md b/README.md index 3fba9e10fb..4293d400bc 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ CVAT annotations ---> Publication, statistics etc. - [MOT sequences](https://arxiv.org/pdf/1906.04567.pdf) - [MOTS PNG](https://www.vision.rwth-aachen.de/page/mots) - [ImageNet](http://image-net.org/) + - [CIFAR-10/100](https://www.cs.toronto.edu/~kriz/cifar.html) (`classification`) - [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [CVAT](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) - [LabelMe](http://labelme.csail.mit.edu/Release3.0) diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py new file mode 100644 index 0000000000..2a74ba99c2 --- /dev/null +++ b/datumaro/plugins/cifar_format.py @@ -0,0 +1,181 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp +import pickle + +import numpy as np +from datumaro.components.converter import Converter +from datumaro.components.extractor import (AnnotationType, DatasetItem, + Importer, Label, LabelCategories, SourceExtractor) +from datumaro.util import cast + + +class CifarPath: + BATCHES_META = 'batches.meta' + TRAIN_ANNOTATION_FILE = 'data_batch_' + IMAGES_DIR = 'images' + IMAGE_SIZE = 32 + +CifarLabel = ['airplane', 'automobile', 'bird', 'cat', + 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] + +# Support for Python version CIFAR-10/100 + +class CifarExtractor(SourceExtractor): + def __init__(self, path, subset=None): + if not osp.isfile(path): + raise FileNotFoundError("Can't read annotation file '%s'" % path) + + if not subset: + file_name = osp.splitext(osp.basename(path))[0] + if file_name.startswith(CifarPath.TRAIN_ANNOTATION_FILE): + subset = 'train_%s' % file_name.split('_')[-1] + else: + subset = file_name.rsplit('_', maxsplit=1)[0] + + super().__init__(subset=subset) + + batches_meta_file = osp.join(osp.dirname(path), CifarPath.BATCHES_META) + self._categories = self._load_categories(batches_meta_file) + + self._items = list(self._load_items(path).values()) + + def _load_categories(self, path): + label_cat = LabelCategories() + + if osp.isfile(path): + # num_cases_per_batch: 1000 + # label_names: ['airplane', 'automobile', 'bird', 'cat', 'deer', + # 'dog', 'frog', 'horse', 'ship', 'truck'] + # num_vis: 3072 + with open(path, 'rb') as labels_file: + data = pickle.load(labels_file) + for label in data['label_names']: + label_cat.add(label) + else: + for label in CifarLabel: + label_cat.add(label) + + return { AnnotationType.label: label_cat } + + def _load_items(self, path): + items = {} + + # 'batch_label': 'training batch 1 of 5' + # 'data': ndarray + # 'filenames': list + # 'labels': list + with open(path, 'rb') as anno_file: + annotation_dict = pickle.load(anno_file) + + labels = annotation_dict.get('labels', []) + filenames = annotation_dict.get('filenames', []) + images_data = annotation_dict.get('data') + size = annotation_dict.get('image_sizes') + + if len(labels) != len(filenames): + raise Exception("The sizes of the arrays 'filenames', " \ + "'labels' don't match.") + + if 0 < len(images_data) and len(images_data) != len(filenames): + raise Exception("The sizes of the arrays 'data', " \ + "'filenames', 'labels' don't match.") + + for i, (filename, label) in enumerate(zip(filenames, labels)): + item_id = osp.splitext(filename)[0] + annotations = [] + if label != None: + annotations.append(Label(label)) + + image = None + if 0 < len(images_data): + image = images_data[i] + if size is not None and image is not None: + image = image.reshape(size[i][0], + size[i][1], 3).astype(np.uint8) + elif image is not None: + image = image.reshape(CifarPath.IMAGE_SIZE, + CifarPath.IMAGE_SIZE, 3).astype(np.uint8) + + items[item_id] = DatasetItem(id=item_id, subset=self._subset, + image=image, annotations=annotations) + + return items + + +class CifarImporter(Importer): + @classmethod + def find_sources(cls, path): + return cls._find_sources_recursive(path, '', 'cifar', + file_filter=lambda p: osp.basename(p) not in + {CifarPath.BATCHES_META, CifarPath.IMAGES_DIR}) + + +class CifarConverter(Converter): + DEFAULT_IMAGE_EXT = '.png' + + def apply(self): + label_categories = self._extractor.categories()[AnnotationType.label] + + label_names = [] + for label in label_categories: + label_names.append(label.name) + labels_dict = { 'label_names': label_names } + batches_meta_file = osp.join(self._save_dir, CifarPath.BATCHES_META) + with open(batches_meta_file, 'wb') as labels_file: + pickle.dump(labels_dict, labels_file) + + for subset_name, subset in self._extractor.subsets().items(): + filenames = [] + labels = [] + data = [] + image_sizes = {} + for item in subset: + filenames.append(item.id + self._find_image_ext(item)) + + anns = [a.label for a in item.annotations + if a.type == AnnotationType.label] + label = None + if anns: + label = anns[0] + labels.append(label) + + if item.has_image and self._save_images: + image = item.image + if not image.has_data: + data.append(None) + else: + image = image.data + data.append(image.reshape(-1).astype(np.uint8)) + if image.shape[0] != CifarPath.IMAGE_SIZE or \ + image.shape[1] != CifarPath.IMAGE_SIZE: + image_sizes[len(data) - 1] = (image.shape[0], image.shape[1]) + + annotation_dict = {} + annotation_dict['filenames'] = filenames + annotation_dict['labels'] = labels + annotation_dict['data'] = np.array(data) + if len(image_sizes): + size = (CifarPath.IMAGE_SIZE, CifarPath.IMAGE_SIZE) + # 'image_sizes' isn't included in the standart format, + # needed for different image sizes + annotation_dict['image_sizes'] = [image_sizes.get(p, size) + for p in range(len(data))] + + filename = '%s_batch' % subset_name + batch_label = None + if subset_name.startswith('train_') and \ + cast(subset_name.split('_')[1], int) is not None: + num = subset_name.split('_')[1] + filename = CifarPath.TRAIN_ANNOTATION_FILE + num + batch_label = 'training batch %s of 5' % (num, ) + if subset_name == 'test': + batch_label = 'testing batch 1 of 1' + if batch_label: + annotation_dict['batch_label'] = batch_label + + annotation_file = osp.join(self._save_dir, filename) + with open(annotation_file, 'wb') as labels_file: + pickle.dump(annotation_dict, labels_file) diff --git a/docs/user_manual.md b/docs/user_manual.md index 17f9afeaca..9930e31394 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -113,6 +113,9 @@ List of supported formats: - [Dataset example](../tests/assets/imagenet_dataset) - [Dataset example (txt for classification)](../tests/assets/imagenet_txt_dataset) - Detection format is the same as in PASCAL VOC +- CIFAR-10/100 (`classification` (python version)) + - [Format specification](https://www.cs.toronto.edu/~kriz/cifar.html) + - [Dataset example](../tests/assets/cifar_dataset) - CamVid (`segmentation`) - [Format specification](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [Dataset example](../tests/assets/camvid_dataset) diff --git a/tests/assets/cifar_dataset/batches.meta b/tests/assets/cifar_dataset/batches.meta new file mode 100644 index 0000000000000000000000000000000000000000..2021e05c912bc5558443209c27a82265af0fc307 GIT binary patch literal 70 zcmZo*nQF}d0ku04UBG5&!@I literal 0 HcmV?d00001 diff --git a/tests/assets/cifar_dataset/data_batch_1 b/tests/assets/cifar_dataset/data_batch_1 new file mode 100644 index 0000000000000000000000000000000000000000..e4ed1edc0b8f4b129ff28008e584bf6f04e1a70b GIT binary patch literal 3317 zcmeI#zfQw25C(7?S`aiVd&W+YGVlVh5s`r%MVG`)EhSFm+CwCeSol-vwjM9ZOMvS> z2KNmf?9Z0`+wN`g{ga58S6buMM)%6Vpmy|3xY7f3s>-v;bOd& zF@BiltQoD!2H!gk*1}AfR8}>k**WVQM+d@XED~Ot_Eb59XWr>v*7QkFv@T>JA4T#( z+iP65Zl2Wf)|t;~|8**yqkN&VEFXSD;!+4l_5MzTA4m=humB6N01L1H3$OqSu)x0& hpfCyGss`6=t2n2G(ZpHMMw>2;+i9MToAj1^_6MrCYlQ#+ literal 0 HcmV?d00001 diff --git a/tests/assets/cifar_dataset/test_batch b/tests/assets/cifar_dataset/test_batch new file mode 100644 index 0000000000000000000000000000000000000000..e3776023189c3f89998c42f2d7ff9c0aa6e7a34d GIT binary patch literal 9494 zcmeI&Jxjzu5C-7QeJGNY+NV&sDiHM-XlF2my@XA&mkT7DkZg*8f{h;s)7|)cz1$ua z{siHjV&NTTws`7!TYS4$!qzLV=xkk9O3PaH{5(astYx8+>!8&I|E8Ni>DDGiCzVW9 zW!EAnnnyE5i(DGXbfJg39fPdtR8S9aaB%Lf(EI2j=9wa2&@EKU3K!5{A?5owyXjK|ufl z5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=|5kvfOCWMn bX|hdXkCm9tjp|LQ3*UDA&~Mg$$V0FP+xLys literal 0 HcmV?d00001 diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py new file mode 100644 index 0000000000..627c0065ab --- /dev/null +++ b/tests/test_cifar_format.py @@ -0,0 +1,149 @@ +import os.path as osp +from unittest import TestCase + +import numpy as np +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import (AnnotationType, DatasetItem, Label, + LabelCategories) +from datumaro.plugins.cifar_format import CifarConverter, CifarImporter +from datumaro.util.image import Image +from datumaro.util.test_utils import TestDir, compare_datasets + + +class CifarFormatTest(TestCase): + def test_can_save_and_load(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='image_2', subset='test', + image=np.ones((32, 32, 3)), + annotations=[Label(0)] + ), + DatasetItem(id='image_3', subset='test', + image=np.ones((32, 32, 3)) + ), + DatasetItem(id='image_4', subset='test', + image=np.ones((32, 32, 3)), + annotations=[Label(1)] + ) + ], categories=['label_0', 'label_1']) + + with TestDir() as test_dir: + CifarConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + + def test_can_save_and_load_without_saving_images(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='a', subset='train_1', + annotations=[Label(0)] + ), + DatasetItem(id='b', subset='train_first', + annotations=[Label(1)] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label' + str(label) for label in range(2)), + }) + + with TestDir() as test_dir: + CifarConverter.convert(source_dataset, test_dir, save_images=False) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + + def test_can_save_and_load_with_different_image_size(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='image_1', + image=np.ones((10, 8, 3)), + annotations=[Label(0)] + ), + DatasetItem(id='image_2', + image=np.ones((32, 32, 3)), + annotations=[Label(1)] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label' + str(label) for label in range(2)), + }) + + with TestDir() as test_dir: + CifarConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id="кириллица с пробелом", + image=np.ones((32, 32, 3)), + annotations=[Label(0)] + ), + ], categories=['label_0']) + + with TestDir() as test_dir: + CifarConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((32, 32, 3)))), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((32, 32, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + CifarConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, dataset, parsed_dataset, + require_images=True) + + def test_can_save_and_load_empty_image(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='a', annotations=[Label(0)]), + DatasetItem(id='b') + ], categories=['label_0']) + + with TestDir() as test_dir: + CifarConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'cifar') + + compare_datasets(self, dataset, parsed_dataset, + require_images=True) + +DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'cifar_dataset') + +class CifarImporterTest(TestCase): + def test_can_import(self): + expected_dataset = Dataset.from_iterable([ + DatasetItem(id='image_1', subset='train_1', + image=np.ones((32, 32, 3)), + annotations=[Label(0)] + ), + DatasetItem(id='image_2', subset='test', + image=np.ones((32, 32, 3)), + annotations=[Label(1)] + ), + DatasetItem(id='image_3', subset='test', + image=np.ones((32, 32, 3)), + annotations=[Label(3)] + ), + DatasetItem(id='image_4', subset='test', + image=np.ones((32, 32, 3)), + annotations=[Label(2)] + ) + ], categories=['airplane', 'automobile', 'bird', 'cat']) + + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'cifar') + + compare_datasets(self, expected_dataset, dataset) + + def test_can_detect(self): + self.assertTrue(CifarImporter.detect(DUMMY_DATASET_DIR)) From ec4b013308534681d23ae3720dde7cdb844aed21 Mon Sep 17 00:00:00 2001 From: Jihyeon Yi Date: Thu, 29 Apr 2021 22:41:49 +0900 Subject: [PATCH 14/40] add validation item for instance segmentation (#227) * add validation item for instance segmentation --- datumaro/cli/contexts/project/__init__.py | 17 +- datumaro/components/errors.py | 35 +- datumaro/components/validator.py | 1157 +++++++++++++-------- tests/test_validator.py | 289 ++++- 4 files changed, 1004 insertions(+), 494 deletions(-) diff --git a/datumaro/cli/contexts/project/__init__.py b/datumaro/cli/contexts/project/__init__.py index 64c4a28481..963f52158e 100644 --- a/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/cli/contexts/project/__init__.py @@ -8,6 +8,7 @@ import os import os.path as osp import shutil +import numpy as np from enum import Enum from datumaro.components.dataset_filter import DatasetItemEncoder @@ -815,7 +816,7 @@ def validate_command(args): project = load_project(args.project_dir) task_type = args.task_type subset_name = args.subset_name - dst_file_name = 'validation_results' + dst_file_name = f'validation_results-{task_type}' dataset = project.make_dataset() if subset_name is not None: @@ -823,20 +824,26 @@ def validate_command(args): dst_file_name += f'-{subset_name}' validation_results = validate_annotations(dataset, task_type) - def _convert_tuple_keys_to_str(d): + def numpy_encoder(obj): + if isinstance(obj, np.generic): + return obj.item() + + def _make_serializable(d): for key, val in list(d.items()): + # tuple key to str if isinstance(key, tuple): d[str(key)] = val d.pop(key) if isinstance(val, dict): - _convert_tuple_keys_to_str(val) + _make_serializable(val) - _convert_tuple_keys_to_str(validation_results) + _make_serializable(validation_results) dst_file = generate_next_file_name(dst_file_name, ext='.json') log.info("Writing project validation results to '%s'" % dst_file) with open(dst_file, 'w') as f: - json.dump(validation_results, f, indent=4, sort_keys=True) + json.dump(validation_results, f, indent=4, sort_keys=True, + default=numpy_encoder) def build_parser(parser_ctor=argparse.ArgumentParser): parser = parser_ctor( diff --git a/datumaro/components/errors.py b/datumaro/components/errors.py index 3d8da0629b..717399b060 100644 --- a/datumaro/components/errors.py +++ b/datumaro/components/errors.py @@ -119,10 +119,13 @@ def __str__(self): return "Metadata (ex. LabelCategories) should be defined" \ " to validate a dataset." + @attrs -class MissingLabelAnnotation(DatasetItemValidationError): +class MissingAnnotation(DatasetItemValidationError): + ann_type = attrib() def __str__(self): - return "Item needs a label, but not found." + return f"Item needs '{self.ann_type}' annotation(s), " \ + "but not found." @attrs class MultiLabelAnnotations(DatasetItemValidationError): @@ -228,32 +231,26 @@ def __str__(self): f" '{self. attr_name}' for the label '{self.label_name}'." @attrs -class ImbalancedBboxDistInLabel(DatasetValidationError): +class ImbalancedDistInLabel(DatasetValidationError): label_name = attrib() prop = attrib() def __str__(self): - return f"Values of bbox '{self.prop}' are not evenly " \ + return f"Values of '{self.prop}' are not evenly " \ f"distributed for '{self.label_name}' label." @attrs -class ImbalancedBboxDistInAttribute(DatasetValidationError): +class ImbalancedDistInAttribute(DatasetValidationError): label_name = attrib() attr_name = attrib() attr_value = attrib() prop = attrib() def __str__(self): - return f"Values of bbox '{self.prop}' are not evenly " \ + return f"Values of '{self.prop}' are not evenly " \ f"distributed for '{self.attr_name}' = '{self.attr_value}' for " \ f"the '{self.label_name}' label." -@attrs -class MissingBboxAnnotation(DatasetItemValidationError): - def __str__(self): - return 'Item needs one or more bounding box annotations, ' \ - 'but not found.' - @attrs class NegativeLength(DatasetItemValidationError): ann_id = attrib() @@ -261,7 +258,7 @@ class NegativeLength(DatasetItemValidationError): val = attrib() def __str__(self): - return f"Bounding box annotation '{self.ann_id}' in " \ + return f"Annotation '{self.ann_id}' in " \ "the item should have a positive value of " \ f"'{self.prop}' but got '{self.val}'." @@ -271,9 +268,9 @@ class InvalidValue(DatasetItemValidationError): prop = attrib() def __str__(self): - return f"Bounding box annotation '{self.ann_id}' in " \ + return f"Annotation '{self.ann_id}' in " \ 'the item has an inf or a NaN value of ' \ - f"bounding box '{self.prop}'." + f"'{self.prop}'." @attrs class FarFromLabelMean(DatasetItemValidationError): @@ -284,8 +281,8 @@ class FarFromLabelMean(DatasetItemValidationError): val = attrib() def __str__(self): - return f"Bounding box annotation '{self.ann_id}' in " \ - f"the item has a value of bounding box '{self.prop}' that " \ + return f"Annotation '{self.ann_id}' in " \ + f"the item has a value of '{self.prop}' that " \ "is too far from the label average. (mean of " \ f"'{self.label_name}' label: {self.mean}, got '{self.val}')." @@ -300,8 +297,8 @@ class FarFromAttrMean(DatasetItemValidationError): val = attrib() def __str__(self): - return f"Bounding box annotation '{self.ann_id}' in the " \ - f"item has a value of bounding box '{self.prop}' that " \ + return f"Annotation '{self.ann_id}' in the " \ + f"item has a value of '{self.prop}' that " \ "is too far from the attribute average. (mean of " \ f"'{self.attr_name}' = '{self.attr_value}' for the " \ f"'{self.label_name}' label: {self.mean}, got '{self.val}')." diff --git a/datumaro/components/validator.py b/datumaro/components/validator.py index dfa524d752..2d03363475 100644 --- a/datumaro/components/validator.py +++ b/datumaro/components/validator.py @@ -10,12 +10,12 @@ from datumaro.components.dataset import IDataset from datumaro.components.errors import (MissingLabelCategories, - MissingLabelAnnotation, MultiLabelAnnotations, MissingAttribute, + MissingAnnotation, MultiLabelAnnotations, MissingAttribute, UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, - ImbalancedBboxDistInLabel, ImbalancedBboxDistInAttribute, - MissingBboxAnnotation, NegativeLength, InvalidValue, FarFromLabelMean, + ImbalancedDistInLabel, ImbalancedDistInAttribute, + NegativeLength, InvalidValue, FarFromLabelMean, FarFromAttrMean, OnlyOneAttributeValue) from datumaro.components.extractor import AnnotationType, LabelCategories from datumaro.util import parse_str_enum_value @@ -23,25 +23,38 @@ Severity = Enum('Severity', ['warning', 'error']) -TaskType = Enum('TaskType', ['classification', 'detection']) +TaskType = Enum('TaskType', ['classification', 'detection', 'segmentation']) class _Validator: DEFAULT_FEW_SAMPLES = 1 DEFAULT_IMBALANCE_RATIO = 50 + DEFAULT_FAR_FROM_MEAN = 5.0 + DEFAULT_DOMINANCE_RATIO = 0.8 + DEFAULT_DOMINANCE_TOPK_BINS = 0.1 + + # statistics templates + numerical_stat_template = { + 'items_far_from_mean': {}, + 'mean': None, + 'stdev': None, + 'min': None, + 'max': None, + 'median': None, + 'histogram': { + 'bins': [], + 'counts': [], + }, + 'distribution': np.array([]) + } + """ A base class for task-specific validators. - ... - Attributes ---------- task_type : str or TaskType - task type (ie. classification, detection etc.) - ann_type : str or AnnotationType - annotation type to validate (default is AnnotationType.label) - far_from_mean_thr : float - constant used to define mean +/- k * stdev (default is None) + task type (ie. classification, detection, segmentation) Methods ------- @@ -51,42 +64,35 @@ class _Validator: Abstract method that must be implemented in a subclass. """ - def __init__(self, task_type=None, ann_type=None, far_from_mean_thr=None): - task_type = parse_str_enum_value(task_type, TaskType, + def __init__(self, task_type=None): + self.task_type = parse_str_enum_value(task_type, TaskType, default=TaskType.classification) - ann_type = parse_str_enum_value(ann_type, AnnotationType, - default=AnnotationType.label) - self.task_type = task_type - self.ann_type = ann_type + if self.task_type == TaskType.classification: + self.ann_types = {AnnotationType.label} + self.str_ann_type = "label" + elif self.task_type == TaskType.detection: + self.ann_types = {AnnotationType.bbox} + self.str_ann_type = "bounding box" + elif self.task_type == TaskType.segmentation: + self.ann_types = {AnnotationType.mask, AnnotationType.polygon} + self.str_ann_type = "mask or polygon" - self.far_from_mean_thr = far_from_mean_thr + self.far_from_mean_thr = self.DEFAULT_FAR_FROM_MEAN self.imbalance_ratio_thr = self.DEFAULT_IMBALANCE_RATIO self.few_samples_thr = self.DEFAULT_FEW_SAMPLES + self.dominance_thr = self.DEFAULT_DOMINANCE_RATIO + self.topk_bins_ratio = self.DEFAULT_DOMINANCE_TOPK_BINS - def compute_statistics(self, dataset): - """ - Computes various statistics of the dataset based on task type. - - Parameters - ---------- - dataset : IDataset object - - Returns - ------- - stats (dict): A dict object containing statistics of the dataset. - """ - + def _compute_common_statistics(self, dataset): defined_attr_template = { 'items_missing_attribute': [], 'distribution': {} } - undefined_attr_template = { 'items_with_undefined_attr': [], 'distribution': {} } - undefined_label_template = { 'count': 0, 'items_with_undefined_label': [], @@ -102,6 +108,8 @@ def compute_statistics(self, dataset): 'undefined_attributes': {} }, } + stats['total_ann_count'] = 0 + stats['items_missing_annotation'] = [] label_dist = stats['label_distribution'] attr_dist = stats['attribute_distribution'] @@ -114,292 +122,129 @@ def compute_statistics(self, dataset): LabelCategories()) base_valid_attrs = label_categories.attributes - if self.task_type == TaskType.classification: - stats['total_label_count'] = 0 - stats['items_missing_label'] = [] - stats['items_with_multiple_labels'] = [] - - elif self.task_type == TaskType.detection: - bbox_info_template = { - 'items_far_from_mean': {}, - 'mean': None, - 'stdev': None, - 'min': None, - 'max': None, - 'median': None, - 'histogram': { - 'bins': [], - 'counts': [], - }, - 'distribution': np.array([]) - } - - bbox_template = { - 'width': deepcopy(bbox_info_template), - 'height': deepcopy(bbox_info_template), - 'area(wxh)': deepcopy(bbox_info_template), - 'ratio(w/h)': deepcopy(bbox_info_template), - 'short': deepcopy(bbox_info_template), - 'long': deepcopy(bbox_info_template) - } - - stats['total_bbox_count'] = 0 - stats['items_missing_bbox'] = [] - stats['items_with_negative_length'] = {} - stats['items_with_invalid_value'] = {} - stats['bbox_distribution_in_label'] = {} - stats['bbox_distribution_in_attribute'] = {} - stats['bbox_distribution_in_dataset_item'] = {} - - bbox_dist_by_label = stats['bbox_distribution_in_label'] - bbox_dist_by_attr = stats['bbox_distribution_in_attribute'] - bbox_dist_in_item = stats['bbox_distribution_in_dataset_item'] - items_w_neg_len = stats['items_with_negative_length'] - items_w_invalid_val = stats['items_with_invalid_value'] - _k = self.far_from_mean_thr - - def _update_prop_distributions(ann_bbox_info, target_stats): - for prop, val in ann_bbox_info.items(): - prop_stats = target_stats[prop] - prop_dist = prop_stats['distribution'] - prop_stats['distribution'] = np.append(prop_dist, val) - - def _generate_ann_bbox_info(_x, _y, _w, _h, area, - ratio, _short, _long): - return { - 'x': _x, - 'y': _y, - 'width': _w, - 'height': _h, - 'area(wxh)': area, - 'ratio(w/h)': ratio, - 'short': _short, - 'long': _long, - } - - def _update_bbox_stats_by_label(item, ann, bbox_label_stats): - bbox_has_error = False - - _x, _y, _w, _h = ann.get_bbox() - area = ann.get_area() - - if _h != 0 and _h != float('inf'): - ratio = _w / _h - else: - ratio = float('nan') - - _short = _w if _w < _h else _h - _long = _w if _w > _h else _h - - ann_bbox_info = _generate_ann_bbox_info( - _x, _y, _w, _h, area, ratio, _short, _long) - - for prop, val in ann_bbox_info.items(): - if val == float('inf') or np.isnan(val): - bbox_has_error = True - anns_w_invalid_val = items_w_invalid_val.setdefault( - (item.id, item.subset), {}) - invalid_props = anns_w_invalid_val.setdefault( - ann.id, []) - invalid_props.append(prop) - - for prop in ['width', 'height']: - val = ann_bbox_info[prop] - if val < 1: - bbox_has_error = True - anns_w_neg_len = items_w_neg_len.setdefault( - (item.id, item.subset), {}) - neg_props = anns_w_neg_len.setdefault(ann.id, {}) - neg_props[prop] = val - - if not bbox_has_error: - ann_bbox_info.pop('x') - ann_bbox_info.pop('y') - _update_prop_distributions(ann_bbox_info, bbox_label_stats) - - return ann_bbox_info, bbox_has_error - - def _compute_prop_stats_from_dist(): - for label_name, bbox_stats in bbox_dist_by_label.items(): - prop_stats_list = list(bbox_stats.values()) - bbox_attr_label = bbox_dist_by_attr.get(label_name, {}) - for vals in bbox_attr_label.values(): - for val_stats in vals.values(): - prop_stats_list += list(val_stats.values()) - - for prop_stats in prop_stats_list: - prop_dist = prop_stats.pop('distribution', []) - if len(prop_dist) > 0: - prop_stats['mean'] = np.mean(prop_dist) - prop_stats['stdev'] = np.std(prop_dist) - prop_stats['min'] = np.min(prop_dist) - prop_stats['max'] = np.max(prop_dist) - prop_stats['median'] = np.median(prop_dist) - - counts, bins = np.histogram(prop_dist) - prop_stats['histogram']['bins'] = bins.tolist() - prop_stats['histogram']['counts'] = counts.tolist() - - def _is_valid_bbox(item, ann): - is_bbox = ann.type == self.ann_type - has_defined_label = 0 <= ann.label < len(label_categories) - if not is_bbox or not has_defined_label: - return False - - bbox_has_neg_len = ann.id in items_w_neg_len.get( - (item.id, item.subset), {}) - bbox_has_invalid_val = ann.id in items_w_invalid_val.get( - (item.id, item.subset), {}) - return not (bbox_has_neg_len or bbox_has_invalid_val) - - def _far_from_mean(val, mean, stdev): - return val > mean + (_k * stdev) or val < mean - (_k * stdev) - - def _update_props_far_from_mean(item, ann): - valid_attrs = base_valid_attrs.union( - label_categories[ann.label].attributes) - label_name = label_categories[ann.label].name - bbox_label_stats = bbox_dist_by_label[label_name] - - _x, _y, _w, _h = ann.get_bbox() - area = ann.get_area() - ratio = _w / _h - _short = _w if _w < _h else _h - _long = _w if _w > _h else _h - - ann_bbox_info = _generate_ann_bbox_info( - _x, _y, _w, _h, area, ratio, _short, _long) - ann_bbox_info.pop('x') - ann_bbox_info.pop('y') - - for prop, val in ann_bbox_info.items(): - prop_stats = bbox_label_stats[prop] - items_far_from_mean = prop_stats['items_far_from_mean'] - mean = prop_stats['mean'] - stdev = prop_stats['stdev'] - - if _far_from_mean(val, mean, stdev): - bboxs_far_from_mean = items_far_from_mean.setdefault( - (item.id, item.subset), {}) - bboxs_far_from_mean[ann.id] = val - - for attr, value in ann.attributes.items(): - if attr in valid_attrs: - bbox_attr_stats = bbox_dist_by_attr[label_name][attr] - bbox_val_stats = bbox_attr_stats[str(value)] - - for prop, val in ann_bbox_info.items(): - prop_stats = bbox_val_stats[prop] - items_far_from_mean = \ - prop_stats['items_far_from_mean'] - mean = prop_stats['mean'] - stdev = prop_stats['stdev'] - - if _far_from_mean(val, mean, stdev): - bboxs_far_from_mean = \ - items_far_from_mean.setdefault( - (item.id, item.subset), {}) - bboxs_far_from_mean[ann.id] = val - for category in label_categories: defined_label_dist[category.name] = 0 + filtered_anns = [] for item in dataset: - ann_count = [ann.type == self.ann_type - for ann in item.annotations].count(True) - - if self.task_type == TaskType.classification: - if ann_count == 0: - stats['items_missing_label'].append((item.id, item.subset)) - elif ann_count > 1: - stats['items_with_multiple_labels'].append( - (item.id, item.subset)) - stats['total_label_count'] += ann_count - - elif self.task_type == TaskType.detection: - if ann_count < 1: - stats['items_missing_bbox'].append((item.id, item.subset)) - stats['total_bbox_count'] += ann_count - bbox_dist_in_item[(item.id, item.subset)] = ann_count - + item_key = (item.id, item.subset) + annotations = [] for ann in item.annotations: - if ann.type == self.ann_type: - if not 0 <= ann.label < len(label_categories): - label_name = ann.label - - label_stats = undefined_label_dist.setdefault( - ann.label, deepcopy(undefined_label_template)) - label_stats['items_with_undefined_label'].append( - (item.id, item.subset)) - - label_stats['count'] += 1 - valid_attrs = set() - missing_attrs = set() - else: - label_name = label_categories[ann.label].name - defined_label_dist[label_name] += 1 + if ann.type in self.ann_types: + annotations.append(ann) + ann_count = len(annotations) + filtered_anns.append((item_key, annotations)) + + if ann_count == 0: + stats['items_missing_annotation'].append(item_key) + stats['total_ann_count'] += ann_count + + for ann in annotations: + if not 0 <= ann.label < len(label_categories): + label_name = ann.label + + label_stats = undefined_label_dist.setdefault( + ann.label, deepcopy(undefined_label_template)) + label_stats['items_with_undefined_label'].append( + item_key) + + label_stats['count'] += 1 + valid_attrs = set() + missing_attrs = set() + else: + label_name = label_categories[ann.label].name + defined_label_dist[label_name] += 1 - defined_attr_stats = defined_attr_dist.setdefault( - label_name, {}) + defined_attr_stats = defined_attr_dist.setdefault( + label_name, {}) - valid_attrs = base_valid_attrs.union( - label_categories[ann.label].attributes) - ann_attrs = getattr(ann, 'attributes', {}).keys() - missing_attrs = valid_attrs.difference(ann_attrs) + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + ann_attrs = getattr(ann, 'attributes', {}).keys() + missing_attrs = valid_attrs.difference(ann_attrs) - for attr in valid_attrs: - defined_attr_stats.setdefault( - attr, deepcopy(defined_attr_template)) + for attr in valid_attrs: + defined_attr_stats.setdefault( + attr, deepcopy(defined_attr_template)) - if self.task_type == TaskType.detection: - bbox_label_stats = bbox_dist_by_label.setdefault( - label_name, deepcopy(bbox_template)) - ann_bbox_info, bbox_has_error = \ - _update_bbox_stats_by_label( - item, ann, bbox_label_stats) + for attr in missing_attrs: + attr_dets = defined_attr_stats[attr] + attr_dets['items_missing_attribute'].append( + item_key) - for attr in missing_attrs: + for attr, value in ann.attributes.items(): + if attr not in valid_attrs: + undefined_attr_stats = \ + undefined_attr_dist.setdefault( + label_name, {}) + attr_dets = undefined_attr_stats.setdefault( + attr, deepcopy(undefined_attr_template)) + attr_dets['items_with_undefined_attr'].append( + item_key) + else: attr_dets = defined_attr_stats[attr] - attr_dets['items_missing_attribute'].append( - (item.id, item.subset)) - - for attr, value in ann.attributes.items(): - if attr not in valid_attrs: - undefined_attr_stats = \ - undefined_attr_dist.setdefault( - label_name, {}) - attr_dets = undefined_attr_stats.setdefault( - attr, deepcopy(undefined_attr_template)) - attr_dets['items_with_undefined_attr'].append( - (item.id, item.subset)) - else: - attr_dets = defined_attr_stats[attr] - - if self.task_type == TaskType.detection and \ - ann.type == self.ann_type: - bbox_attr_label = bbox_dist_by_attr.setdefault( - label_name, {}) - bbox_attr_stats = bbox_attr_label.setdefault( - attr, {}) - bbox_val_stats = bbox_attr_stats.setdefault( - str(value), deepcopy(bbox_template)) - - if not bbox_has_error: - _update_prop_distributions( - ann_bbox_info, bbox_val_stats) - - attr_dets['distribution'].setdefault(str(value), 0) - attr_dets['distribution'][str(value)] += 1 - - if self.task_type == TaskType.detection: - _compute_prop_stats_from_dist() - - for item in dataset: - for ann in item.annotations: - if _is_valid_bbox(item, ann): - _update_props_far_from_mean(item, ann) - return stats + attr_dets['distribution'].setdefault(str(value), 0) + attr_dets['distribution'][str(value)] += 1 + + return stats, filtered_anns + + @staticmethod + def _update_prop_distributions(curr_prop_stats, target_stats): + for prop, val in curr_prop_stats.items(): + prop_stats = target_stats[prop] + prop_dist = prop_stats['distribution'] + prop_stats['distribution'] = np.append(prop_dist, val) + + @staticmethod + def _compute_prop_stats_from_dist(dist_by_label, dist_by_attr): + for label_name, stats in dist_by_label.items(): + prop_stats_list = list(stats.values()) + attr_label = dist_by_attr.get(label_name, {}) + for vals in attr_label.values(): + for val_stats in vals.values(): + prop_stats_list += list(val_stats.values()) + + for prop_stats in prop_stats_list: + prop_dist = prop_stats.pop('distribution', []) + if len(prop_dist) > 0: + prop_stats['mean'] = np.mean(prop_dist) + prop_stats['stdev'] = np.std(prop_dist) + prop_stats['min'] = np.min(prop_dist) + prop_stats['max'] = np.max(prop_dist) + prop_stats['median'] = np.median(prop_dist) + + counts, bins = np.histogram(prop_dist) + prop_stats['histogram']['bins'] = bins.tolist() + prop_stats['histogram']['counts'] = counts.tolist() + + def _compute_far_from_mean(self, prop_stats, val, item_key, ann): + def _far_from_mean(val, mean, stdev): + thr = self.far_from_mean_thr + return val > mean + (thr * stdev) or val < mean - (thr * stdev) + + mean = prop_stats['mean'] + stdev = prop_stats['stdev'] + + if _far_from_mean(val, mean, stdev): + items_far_from_mean = prop_stats['items_far_from_mean'] + far_from_mean = items_far_from_mean.setdefault( + item_key, {}) + far_from_mean[ann.id] = val + + def compute_statistics(self, dataset): + """ + Computes statistics of the dataset based on task type. + + Parameters + ---------- + dataset : IDataset object + + Returns + ------- + stats (dict): A dict object containing statistics of the dataset. + """ + return NotImplementedError def _check_missing_label_categories(self, stats): validation_reports = [] @@ -410,6 +255,17 @@ def _check_missing_label_categories(self, stats): return validation_reports + def _check_missing_annotation(self, stats): + validation_reports = [] + + items_missing = stats['items_missing_annotation'] + for item_id, item_subset in items_missing: + validation_reports += self._generate_validation_report( + MissingAnnotation, Severity.warning, item_id, item_subset, + self.str_ann_type) + + return validation_reports + def _check_missing_attribute(self, label_name, attr_name, attr_dets): validation_reports = [] @@ -561,6 +417,114 @@ def _check_imbalanced_attribute(self, label_name, attr_name, attr_dets): return validation_reports + def _check_imbalanced_dist_in_label(self, label_name, label_stats): + validation_reports = [] + thr = self.dominance_thr + topk_ratio = self.topk_bins_ratio + + for prop, prop_stats in label_stats.items(): + value_counts = prop_stats['histogram']['counts'] + n_bucket = len(value_counts) + if n_bucket < 2: + continue + topk = max(1, int(np.around(n_bucket * topk_ratio))) + + if topk > 0: + topk_values = np.sort(value_counts)[-topk:] + ratio = np.sum(topk_values) / np.sum(value_counts) + if ratio >= thr: + details = (label_name, f"{self.str_ann_type} {prop}") + validation_reports += self._generate_validation_report( + ImbalancedDistInLabel, Severity.warning, *details) + + return validation_reports + + def _check_imbalanced_dist_in_attr(self, label_name, attr_name, attr_stats): + validation_reports = [] + thr = self.dominance_thr + topk_ratio = self.topk_bins_ratio + + for attr_value, value_stats in attr_stats.items(): + for prop, prop_stats in value_stats.items(): + value_counts = prop_stats['histogram']['counts'] + n_bucket = len(value_counts) + if n_bucket < 2: + continue + topk = max(1, int(np.around(n_bucket * topk_ratio))) + + if topk > 0: + topk_values = np.sort(value_counts)[-topk:] + ratio = np.sum(topk_values) / np.sum(value_counts) + if ratio >= thr: + details = (label_name, attr_name, attr_value, + f"{self.str_ann_type} {prop}") + validation_reports += self._generate_validation_report( + ImbalancedDistInAttribute, + Severity.warning, + *details + ) + + return validation_reports + + def _check_invalid_value(self, stats): + validation_reports = [] + + items_w_invalid_val = stats['items_with_invalid_value'] + for item_dets, anns_w_invalid_val in items_w_invalid_val.items(): + item_id, item_subset = item_dets + for ann_id, props in anns_w_invalid_val.items(): + for prop in props: + details = (item_subset, ann_id, + f"{self.str_ann_type} {prop}") + validation_reports += self._generate_validation_report( + InvalidValue, Severity.error, item_id, *details) + + return validation_reports + + def _check_far_from_label_mean(self, label_name, label_stats): + validation_reports = [] + + for prop, prop_stats in label_stats.items(): + items_far_from_mean = prop_stats['items_far_from_mean'] + if prop_stats['mean'] is not None: + mean = round(prop_stats['mean'], 2) + + for item_dets, anns_far in items_far_from_mean.items(): + item_id, item_subset = item_dets + for ann_id, val in anns_far.items(): + val = round(val, 2) + details = (item_subset, label_name, ann_id, + f"{self.str_ann_type} {prop}", mean, val) + validation_reports += self._generate_validation_report( + FarFromLabelMean, Severity.warning, item_id, *details) + + return validation_reports + + def _check_far_from_attr_mean(self, label_name, attr_name, attr_stats): + validation_reports = [] + + for attr_value, value_stats in attr_stats.items(): + for prop, prop_stats in value_stats.items(): + items_far_from_mean = prop_stats['items_far_from_mean'] + if prop_stats['mean'] is not None: + mean = round(prop_stats['mean'], 2) + + for item_dets, anns_far in items_far_from_mean.items(): + item_id, item_subset = item_dets + for ann_id, val in anns_far.items(): + val = round(val, 2) + details = (item_subset, label_name, ann_id, attr_name, + attr_value, f"{self.str_ann_type} {prop}", + mean, val) + validation_reports += self._generate_validation_report( + FarFromAttrMean, + Severity.warning, + item_id, + *details + ) + + return validation_reports + def generate_reports(self, stats): raise NotImplementedError('Should be implemented in a subclass.') @@ -574,17 +538,7 @@ class ClassificationValidator(_Validator): """ def __init__(self): - super().__init__(TaskType.classification, AnnotationType.label) - - def _check_missing_label_annotation(self, stats): - validation_reports = [] - - items_missing_label = stats['items_missing_label'] - for item_id, item_subset in items_missing_label: - validation_reports += self._generate_validation_report( - MissingLabelAnnotation, Severity.warning, item_id, item_subset) - - return validation_reports + super().__init__(TaskType.classification) def _check_multi_label_annotations(self, stats): validation_reports = [] @@ -596,6 +550,30 @@ def _check_multi_label_annotations(self, stats): return validation_reports + def compute_statistics(self, dataset): + """ + Computes statistics of the dataset for the classification task. + + Parameters + ---------- + dataset : IDataset object + + Returns + ------- + stats (dict): A dict object containing statistics of the dataset. + """ + + stats, filtered_anns = self._compute_common_statistics(dataset) + + stats['items_with_multiple_labels'] = [] + + for item_key, anns in filtered_anns: + ann_count = len(anns) + if ann_count > 1: + stats['items_with_multiple_labels'].append(item_key) + + return stats + def generate_reports(self, stats): """ Validates the dataset for classification tasks based on its statistics. @@ -613,7 +591,7 @@ def generate_reports(self, stats): reports = [] reports += self._check_missing_label_categories(stats) - reports += self._check_missing_label_annotation(stats) + reports += self._check_missing_annotation(stats) reports += self._check_multi_label_annotations(stats) reports += self._check_label_defined_but_not_found(stats) reports += self._check_only_one_label(stats) @@ -658,76 +636,8 @@ class DetectionValidator(_Validator): """ A validator class for detection tasks. """ - - DEFAULT_FAR_FROM_MEAN = 5.0 - DEFAULT_BBOX_IMBALANCE = 0.8 - DEFAULT_BBOX_TOPK_BINS = 0.1 - def __init__(self): - super().__init__(TaskType.detection, AnnotationType.bbox, - far_from_mean_thr=self.DEFAULT_FAR_FROM_MEAN) - self.bbox_imbalance_thr = self.DEFAULT_BBOX_IMBALANCE - self.bbox_topk_bins_ratio = self.DEFAULT_BBOX_TOPK_BINS - - def _check_imbalanced_bbox_dist_in_label(self, label_name, - bbox_label_stats): - validation_reports = [] - thr = self.bbox_imbalance_thr - topk_ratio = self.bbox_topk_bins_ratio - - for prop, prop_stats in bbox_label_stats.items(): - value_counts = prop_stats['histogram']['counts'] - n_bucket = len(value_counts) - if n_bucket < 2: - continue - topk = max(1, int(np.around(n_bucket * topk_ratio))) - - if topk > 0: - topk_values = np.sort(value_counts)[-topk:] - ratio = np.sum(topk_values) / np.sum(value_counts) - if ratio >= thr: - details = (label_name, prop) - validation_reports += self._generate_validation_report( - ImbalancedBboxDistInLabel, Severity.warning, *details) - - return validation_reports - - def _check_imbalanced_bbox_dist_in_attr(self, label_name, attr_name, - bbox_attr_stats): - validation_reports = [] - thr = self.bbox_imbalance_thr - topk_ratio = self.bbox_topk_bins_ratio - - for attr_value, value_stats in bbox_attr_stats.items(): - for prop, prop_stats in value_stats.items(): - value_counts = prop_stats['histogram']['counts'] - n_bucket = len(value_counts) - if n_bucket < 2: - continue - topk = max(1, int(np.around(n_bucket * topk_ratio))) - - if topk > 0: - topk_values = np.sort(value_counts)[-topk:] - ratio = np.sum(topk_values) / np.sum(value_counts) - if ratio >= thr: - details = (label_name, attr_name, attr_value, prop) - validation_reports += self._generate_validation_report( - ImbalancedBboxDistInAttribute, - Severity.warning, - *details - ) - - return validation_reports - - def _check_missing_bbox_annotation(self, stats): - validation_reports = [] - - items_missing_bbox = stats['items_missing_bbox'] - for item_id, item_subset in items_missing_bbox: - validation_reports += self._generate_validation_report( - MissingBboxAnnotation, Severity.warning, item_id, item_subset) - - return validation_reports + super().__init__(TaskType.detection) def _check_negative_length(self, stats): validation_reports = [] @@ -738,67 +648,193 @@ def _check_negative_length(self, stats): for ann_id, props in anns_w_neg_len.items(): for prop, val in props.items(): val = round(val, 2) - details = (item_subset, ann_id, prop, val) + details = (item_subset, ann_id, + f"{self.str_ann_type} {prop}", val) validation_reports += self._generate_validation_report( NegativeLength, Severity.error, item_id, *details) return validation_reports - def _check_invalid_value(self, stats): - validation_reports = [] + def compute_statistics(self, dataset): + """ + Computes statistics of the dataset for the detection task. + + Parameters + ---------- + dataset : IDataset object + Returns + ------- + stats (dict): A dict object containing statistics of the dataset. + """ + + stats, filtered_anns = self._compute_common_statistics(dataset) + + # detection-specific + bbox_template = { + 'width': deepcopy(self.numerical_stat_template), + 'height': deepcopy(self.numerical_stat_template), + 'area(wxh)': deepcopy(self.numerical_stat_template), + 'ratio(w/h)': deepcopy(self.numerical_stat_template), + 'short': deepcopy(self.numerical_stat_template), + 'long': deepcopy(self.numerical_stat_template) + } + + stats['items_with_negative_length'] = {} + stats['items_with_invalid_value'] = {} + stats['bbox_distribution_in_label'] = {} + stats['bbox_distribution_in_attribute'] = {} + stats['bbox_distribution_in_dataset_item'] = {} + + dist_by_label = stats['bbox_distribution_in_label'] + dist_by_attr = stats['bbox_distribution_in_attribute'] + bbox_dist_in_item = stats['bbox_distribution_in_dataset_item'] + items_w_neg_len = stats['items_with_negative_length'] items_w_invalid_val = stats['items_with_invalid_value'] - for item_dets, anns_w_invalid_val in items_w_invalid_val.items(): - item_id, item_subset = item_dets - for ann_id, props in anns_w_invalid_val.items(): - for prop in props: - details = (item_subset, ann_id, prop) - validation_reports += self._generate_validation_report( - InvalidValue, Severity.error, item_id, *details) - return validation_reports + def _generate_ann_bbox_info(_x, _y, _w, _h, area, + ratio, _short, _long): + return { + 'x': _x, + 'y': _y, + 'width': _w, + 'height': _h, + 'area(wxh)': area, + 'ratio(w/h)': ratio, + 'short': _short, + 'long': _long, + } - def _check_far_from_label_mean(self, label_name, bbox_label_stats): - validation_reports = [] + def _update_bbox_stats_by_label(item_key, ann, bbox_label_stats): + bbox_has_error = False - for prop, prop_stats in bbox_label_stats.items(): - items_far_from_mean = prop_stats['items_far_from_mean'] - if prop_stats['mean'] is not None: - mean = round(prop_stats['mean'], 2) + _x, _y, _w, _h = ann.get_bbox() + area = ann.get_area() - for item_dets, anns_far in items_far_from_mean.items(): - item_id, item_subset = item_dets - for ann_id, val in anns_far.items(): - val = round(val, 2) - details = (item_subset, label_name, ann_id, prop, mean, val) - validation_reports += self._generate_validation_report( - FarFromLabelMean, Severity.warning, item_id, *details) + if _h != 0 and _h != float('inf'): + ratio = _w / _h + else: + ratio = float('nan') + + _short = _w if _w < _h else _h + _long = _w if _w > _h else _h + + ann_bbox_info = _generate_ann_bbox_info( + _x, _y, _w, _h, area, ratio, _short, _long) + + for prop, val in ann_bbox_info.items(): + if val == float('inf') or np.isnan(val): + bbox_has_error = True + anns_w_invalid_val = items_w_invalid_val.setdefault( + item_key, {}) + invalid_props = anns_w_invalid_val.setdefault( + ann.id, []) + invalid_props.append(prop) + + for prop in ['width', 'height']: + val = ann_bbox_info[prop] + if val < 1: + bbox_has_error = True + anns_w_neg_len = items_w_neg_len.setdefault( + item_key, {}) + neg_props = anns_w_neg_len.setdefault(ann.id, {}) + neg_props[prop] = val + + if not bbox_has_error: + ann_bbox_info.pop('x') + ann_bbox_info.pop('y') + self._update_prop_distributions(ann_bbox_info, bbox_label_stats) - return validation_reports + return ann_bbox_info, bbox_has_error - def _check_far_from_attr_mean(self, label_name, attr_name, bbox_attr_stats): - validation_reports = [] + label_categories = dataset.categories().get(AnnotationType.label, + LabelCategories()) + base_valid_attrs = label_categories.attributes - for attr_value, value_stats in bbox_attr_stats.items(): - for prop, prop_stats in value_stats.items(): - items_far_from_mean = prop_stats['items_far_from_mean'] - if prop_stats['mean'] is not None: - mean = round(prop_stats['mean'], 2) + for item_key, annotations in filtered_anns: + ann_count = len(annotations) - for item_dets, anns_far in items_far_from_mean.items(): - item_id, item_subset = item_dets - for ann_id, val in anns_far.items(): - val = round(val, 2) - details = (item_subset, label_name, ann_id, attr_name, - attr_value, prop, mean, val) - validation_reports += self._generate_validation_report( - FarFromAttrMean, - Severity.warning, - item_id, - *details - ) + bbox_dist_in_item[item_key] = ann_count - return validation_reports + for ann in annotations: + if not 0 <= ann.label < len(label_categories): + label_name = ann.label + valid_attrs = set() + else: + label_name = label_categories[ann.label].name + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + + bbox_label_stats = dist_by_label.setdefault( + label_name, deepcopy(bbox_template)) + ann_bbox_info, bbox_has_error = \ + _update_bbox_stats_by_label( + item_key, ann, bbox_label_stats) + + for attr, value in ann.attributes.items(): + if attr in valid_attrs: + bbox_attr_label = dist_by_attr.setdefault( + label_name, {}) + bbox_attr_stats = bbox_attr_label.setdefault( + attr, {}) + bbox_val_stats = bbox_attr_stats.setdefault( + str(value), deepcopy(bbox_template)) + + if not bbox_has_error: + self._update_prop_distributions( + ann_bbox_info, bbox_val_stats) + + # Compute prop stats from distribution + self._compute_prop_stats_from_dist(dist_by_label, dist_by_attr) + + def _is_valid_ann(item_key, ann): + has_defined_label = 0 <= ann.label < len(label_categories) + if not has_defined_label: + return False + + bbox_has_neg_len = ann.id in items_w_neg_len.get( + item_key, {}) + bbox_has_invalid_val = ann.id in items_w_invalid_val.get( + item_key, {}) + return not (bbox_has_neg_len or bbox_has_invalid_val) + + def _update_props_far_from_mean(item_key, ann): + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + label_name = label_categories[ann.label].name + bbox_label_stats = dist_by_label[label_name] + + _x, _y, _w, _h = ann.get_bbox() + area = ann.get_area() + ratio = _w / _h + _short = _w if _w < _h else _h + _long = _w if _w > _h else _h + + ann_bbox_info = _generate_ann_bbox_info( + _x, _y, _w, _h, area, ratio, _short, _long) + ann_bbox_info.pop('x') + ann_bbox_info.pop('y') + + for prop, val in ann_bbox_info.items(): + prop_stats = bbox_label_stats[prop] + self._compute_far_from_mean(prop_stats, val, item_key, ann) + + for attr, value in ann.attributes.items(): + if attr in valid_attrs: + bbox_attr_stats = dist_by_attr[label_name][attr] + bbox_val_stats = bbox_attr_stats[str(value)] + + for prop, val in ann_bbox_info.items(): + prop_stats = bbox_val_stats[prop] + self._compute_far_from_mean(prop_stats, val, + item_key, ann) + + for item_key, annotations in filtered_anns: + for ann in annotations: + if _is_valid_ann(item_key, ann): + _update_props_far_from_mean(item_key, ann) + + return stats def generate_reports(self, stats): """ @@ -817,7 +853,7 @@ def generate_reports(self, stats): reports = [] reports += self._check_missing_label_categories(stats) - reports += self._check_missing_bbox_annotation(stats) + reports += self._check_missing_annotation(stats) reports += self._check_label_defined_but_not_found(stats) reports += self._check_only_one_label(stats) reports += self._check_few_samples_in_label(stats) @@ -831,8 +867,8 @@ def generate_reports(self, stats): undefined_label_dist = label_dist['undefined_labels'] undefined_attr_dist = attr_dist['undefined_attributes'] - bbox_dist_by_label = stats['bbox_distribution_in_label'] - bbox_dist_by_attr = stats['bbox_distribution_in_attribute'] + dist_by_label = stats['bbox_distribution_in_label'] + dist_by_attr = stats['bbox_distribution_in_attribute'] defined_labels = defined_attr_dist.keys() for label_name in defined_labels: @@ -851,18 +887,18 @@ def generate_reports(self, stats): reports += self._check_missing_attribute( label_name, attr_name, attr_dets) - bbox_label_stats = bbox_dist_by_label[label_name] - bbox_attr_label = bbox_dist_by_attr.get(label_name, {}) + bbox_label_stats = dist_by_label[label_name] + bbox_attr_label = dist_by_attr.get(label_name, {}) reports += self._check_far_from_label_mean( label_name, bbox_label_stats) - reports += self._check_imbalanced_bbox_dist_in_label( + reports += self._check_imbalanced_dist_in_label( label_name, bbox_label_stats) for attr_name, bbox_attr_stats in bbox_attr_label.items(): reports += self._check_far_from_attr_mean( label_name, attr_name, bbox_attr_stats) - reports += self._check_imbalanced_bbox_dist_in_attr( + reports += self._check_imbalanced_dist_in_attr( label_name, attr_name, bbox_attr_stats) for label_name, label_stats in undefined_label_dist.items(): @@ -876,6 +912,243 @@ def generate_reports(self, stats): return reports +class SegmentationValidator(_Validator): + """ + A validator class for (instance) segmentation tasks. + """ + + def __init__(self): + super().__init__(TaskType.segmentation) + + def compute_statistics(self, dataset): + """ + Computes statistics of the dataset for the segmentation task. + + Parameters + ---------- + dataset : IDataset object + + Returns + ------- + stats (dict): A dict object containing statistics of the dataset. + """ + + stats, filtered_anns = self._compute_common_statistics(dataset) + + # segmentation-specific + mask_template = { + 'area': deepcopy(self.numerical_stat_template), + 'width': deepcopy(self.numerical_stat_template), + 'height': deepcopy(self.numerical_stat_template) + } + + stats['items_with_invalid_value'] = {} + stats['mask_distribution_in_label'] = {} + stats['mask_distribution_in_attribute'] = {} + stats['mask_distribution_in_dataset_item'] = {} + + dist_by_label = stats['mask_distribution_in_label'] + dist_by_attr = stats['mask_distribution_in_attribute'] + mask_dist_in_item = stats['mask_distribution_in_dataset_item'] + items_w_invalid_val = stats['items_with_invalid_value'] + + def _generate_ann_mask_info(area, _w, _h): + return { + 'area': area, + 'width': _w, + 'height': _h, + } + + def _update_mask_stats_by_label(item_key, ann, mask_label_stats): + mask_has_error = False + + _x, _y, _w, _h = ann.get_bbox() + + # Detete the following block when #226 is resolved + # https://github.com/openvinotoolkit/datumaro/issues/226 + if ann.type == AnnotationType.mask: + _w += 1 + _h += 1 + + area = ann.get_area() + + ann_mask_info = _generate_ann_mask_info(area, _w, _h) + + for prop, val in ann_mask_info.items(): + if val == float('inf') or np.isnan(val): + mask_has_error = True + anns_w_invalid_val = items_w_invalid_val.setdefault( + item_key, {}) + invalid_props = anns_w_invalid_val.setdefault( + ann.id, []) + invalid_props.append(prop) + + if not mask_has_error: + self._update_prop_distributions(ann_mask_info, mask_label_stats) + + return ann_mask_info, mask_has_error + + label_categories = dataset.categories().get(AnnotationType.label, + LabelCategories()) + base_valid_attrs = label_categories.attributes + + for item_key, annotations in filtered_anns: + ann_count = len(annotations) + mask_dist_in_item[item_key] = ann_count + + for ann in annotations: + if not 0 <= ann.label < len(label_categories): + label_name = ann.label + valid_attrs = set() + else: + label_name = label_categories[ann.label].name + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + + mask_label_stats = dist_by_label.setdefault( + label_name, deepcopy(mask_template)) + ann_mask_info, mask_has_error = \ + _update_mask_stats_by_label( + item_key, ann, mask_label_stats) + + for attr, value in ann.attributes.items(): + if attr in valid_attrs: + mask_attr_label = dist_by_attr.setdefault( + label_name, {}) + mask_attr_stats = mask_attr_label.setdefault( + attr, {}) + mask_val_stats = mask_attr_stats.setdefault( + str(value), deepcopy(mask_template)) + + if not mask_has_error: + self._update_prop_distributions( + ann_mask_info, mask_val_stats) + + # compute prop stats from dist. + self._compute_prop_stats_from_dist(dist_by_label, dist_by_attr) + + def _is_valid_ann(item_key, ann): + has_defined_label = 0 <= ann.label < len(label_categories) + if not has_defined_label: + return False + + mask_has_invalid_val = ann.id in items_w_invalid_val.get( + item_key, {}) + return not mask_has_invalid_val + + def _update_props_far_from_mean(item_key, ann): + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + label_name = label_categories[ann.label].name + mask_label_stats = dist_by_label[label_name] + + _x, _y, _w, _h = ann.get_bbox() + + # Detete the following block when #226 is resolved + # https://github.com/openvinotoolkit/datumaro/issues/226 + if ann.type == AnnotationType.mask: + _w += 1 + _h += 1 + area = ann.get_area() + + ann_mask_info = _generate_ann_mask_info(area, _w, _h) + + for prop, val in ann_mask_info.items(): + prop_stats = mask_label_stats[prop] + self._compute_far_from_mean(prop_stats, val, item_key, ann) + + for attr, value in ann.attributes.items(): + if attr in valid_attrs: + mask_attr_stats = dist_by_attr[label_name][attr] + mask_val_stats = mask_attr_stats[str(value)] + + for prop, val in ann_mask_info.items(): + prop_stats = mask_val_stats[prop] + self._compute_far_from_mean(prop_stats, val, + item_key, ann) + + for item_key, annotations in filtered_anns: + for ann in annotations: + if _is_valid_ann(item_key, ann): + _update_props_far_from_mean(item_key, ann) + + return stats + + def generate_reports(self, stats): + """ + Validates the dataset for segmentation tasks based on its statistics. + + Parameters + ---------- + dataset : IDataset object + stats : Dict object + + Returns + ------- + reports (list): List of validation reports (DatasetValidationError). + """ + + reports = [] + + reports += self._check_missing_label_categories(stats) + reports += self._check_missing_annotation(stats) + reports += self._check_label_defined_but_not_found(stats) + reports += self._check_only_one_label(stats) + reports += self._check_few_samples_in_label(stats) + reports += self._check_imbalanced_labels(stats) + reports += self._check_invalid_value(stats) + + label_dist = stats['label_distribution'] + attr_dist = stats['attribute_distribution'] + defined_attr_dist = attr_dist['defined_attributes'] + undefined_label_dist = label_dist['undefined_labels'] + undefined_attr_dist = attr_dist['undefined_attributes'] + + dist_by_label = stats['mask_distribution_in_label'] + dist_by_attr = stats['mask_distribution_in_attribute'] + + defined_labels = defined_attr_dist.keys() + for label_name in defined_labels: + attr_stats = defined_attr_dist[label_name] + + reports += self._check_attribute_defined_but_not_found( + label_name, attr_stats) + + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_few_samples_in_attribute( + label_name, attr_name, attr_dets) + reports += self._check_imbalanced_attribute( + label_name, attr_name, attr_dets) + reports += self._check_only_one_attribute_value( + label_name, attr_name, attr_dets) + reports += self._check_missing_attribute( + label_name, attr_name, attr_dets) + + mask_label_stats = dist_by_label[label_name] + mask_attr_label = dist_by_attr.get(label_name, {}) + + reports += self._check_far_from_label_mean( + label_name, mask_label_stats) + reports += self._check_imbalanced_dist_in_label( + label_name, mask_label_stats) + + for attr_name, mask_attr_stats in mask_attr_label.items(): + reports += self._check_far_from_attr_mean( + label_name, attr_name, mask_attr_stats) + reports += self._check_imbalanced_dist_in_attr( + label_name, attr_name, mask_attr_stats) + + for label_name, label_stats in undefined_label_dist.items(): + reports += self._check_undefined_label(label_name, label_stats) + + for label_name, attr_stats in undefined_attr_dist.items(): + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_undefined_attribute( + label_name, attr_name, attr_dets) + + return reports + + def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): """ Returns the validation results of a dataset based on task type. @@ -883,7 +1156,7 @@ def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): Args: dataset (IDataset): Dataset to be validated task_type (str or TaskType): Type of the task - (classification, detection etc.) + (classification, detection, segmentation) Raises: ValueError @@ -901,6 +1174,8 @@ def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): validator = ClassificationValidator() elif task_type == TaskType.detection: validator = DetectionValidator() + elif task_type == TaskType.segmentation: + validator = SegmentationValidator() if not isinstance(dataset, IDataset): raise TypeError("Invalid dataset type '%s'" % type(dataset)) diff --git a/tests/test_validator.py b/tests/test_validator.py index d6c8700d80..7229efe741 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -8,16 +8,17 @@ from datumaro.components.dataset import Dataset, DatasetItem from datumaro.components.errors import (MissingLabelCategories, - MissingLabelAnnotation, MultiLabelAnnotations, MissingAttribute, + MissingAnnotation, MultiLabelAnnotations, MissingAttribute, UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, - ImbalancedBboxDistInLabel, ImbalancedBboxDistInAttribute, - MissingBboxAnnotation, NegativeLength, InvalidValue, FarFromLabelMean, + ImbalancedDistInLabel, ImbalancedDistInAttribute, + NegativeLength, InvalidValue, FarFromLabelMean, FarFromAttrMean, OnlyOneAttributeValue) -from datumaro.components.extractor import Bbox, Label +from datumaro.components.extractor import Bbox, Label, Mask, Polygon from datumaro.components.validator import (ClassificationValidator, - DetectionValidator, TaskType, validate_annotations, _Validator) + DetectionValidator, TaskType, validate_annotations, _Validator, + SegmentationValidator) class TestValidatorTemplate(TestCase): @@ -29,12 +30,22 @@ def setUpClass(cls): Bbox(1, 2, 3, 4, id=1, label=0, attributes={ 'a': 1, 'b': 2, }), + Mask(id=2, label=0, attributes={'a': 1, 'b': 2}, + image=np.array([[0, 0, 0, 0, 0], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + ])), ]), DatasetItem(id=2, image=np.ones((2, 4, 3)), annotations=[ Label(2, id=0, attributes={'a': 2, 'b': 2, }), Bbox(2, 3, 1, 4, id=1, label=0, attributes={ 'a': 1, 'b': 1, }), + Mask(id=2, label=0, attributes={'a': 1, 'b': 1}, + image=np.array([[1, 1, 1, 1], [0, 0, 0, 0]]) + ), ]), DatasetItem(id=3), DatasetItem(id=4, image=np.ones((2, 4, 3)), annotations=[ @@ -46,30 +57,54 @@ def setUpClass(cls): Bbox(3, 1, 4, 2, id=3, label=0, attributes={ 'a': 2, 'b': 2, }), + Polygon([1, 3, 1, 5, 5, 5, 5, 3], label=0, id=4, + attributes={'a': 2, 'b': 2, + }), + Polygon([3, 1, 3, 5, 5, 5, 5, 1], label=1, id=5, + attributes={'a': 2, 'b': 1, + }), ]), DatasetItem(id=5, image=np.ones((2, 4, 3)), annotations=[ Label(0, id=0, attributes={'a': 20, 'b': 10, }), Bbox(1, 2, 3, 4, id=1, label=1, attributes={ 'a': 1, 'b': 1, }), + Polygon([1, 2, 1, 5, 5, 5, 5, 2], label=1, id=2, + attributes={'a': 1, 'b': 1, + }), ]), DatasetItem(id=6, image=np.ones((2, 4, 3)), annotations=[ Label(1, id=0, attributes={'a': 11, 'b': 2, 'c': 3, }), Bbox(2, 3, 4, 1, id=1, label=1, attributes={ 'a': 2, 'b': 2, }), + Mask(id=2, label=1, attributes={'a': 2, 'b': 2}, + image=np.array([[1, 0, 0], + [1, 0, 0], + [1, 0, 0], + [1, 0, 0], + ])), ]), DatasetItem(id=7, image=np.ones((2, 4, 3)), annotations=[ Label(1, id=0, attributes={'a': 1, 'b': 2, 'c': 5, }), Bbox(1, 2, 3, 4, id=1, label=2, attributes={ 'a': 1, 'b': 2, }), + Polygon([1, 2, 1, 5, 5, 5, 5, 2], label=2, id=2, + attributes={'a': 1, 'b': 2, + }), ]), DatasetItem(id=8, image=np.ones((2, 4, 3)), annotations=[ Label(2, id=0, attributes={'a': 7, 'b': 9, 'c': 5, }), Bbox(2, 1, 3, 4, id=1, label=2, attributes={ 'a': 2, 'b': 1, }), + Mask(id=2, label=2, attributes={'a': 2, 'b': 1}, + image=np.array([[1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + ])), ]), ], categories=[[f'label_{i}', None, {'a', 'b', }] for i in range(2)]) @@ -319,13 +354,13 @@ def setUpClass(cls): def test_check_missing_label_annotation(self): stats = { - 'items_missing_label': [(1, 'unittest')] + 'items_missing_annotation': [(1, 'unittest')] } - actual_reports = self.validator._check_missing_label_annotation(stats) + actual_reports = self.validator._check_missing_annotation(stats) self.assertTrue(len(actual_reports) == 1) - self.assertIsInstance(actual_reports[0], MissingLabelAnnotation) + self.assertIsInstance(actual_reports[0], MissingAnnotation) def test_check_multi_label_annotations(self): stats = { @@ -343,9 +378,9 @@ class TestDetectionValidator(TestValidatorTemplate): def setUpClass(cls): cls.validator = DetectionValidator() - def test_check_imbalanced_bbox_dist_in_label(self): + def test_check_imbalanced_dist_in_label(self): label_name = 'unittest' - most = int(self.validator.DEFAULT_BBOX_IMBALANCE * 100) + most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -356,11 +391,11 @@ def test_check_imbalanced_bbox_dist_in_label(self): } } } - reports = self.validator._check_imbalanced_bbox_dist_in_label( + reports = self.validator._check_imbalanced_dist_in_label( label_name, bbox_label_stats) self.assertTrue(len(reports) == 1) - self.assertIsInstance(reports[0], ImbalancedBboxDistInLabel) + self.assertIsInstance(reports[0], ImbalancedDistInLabel) with self.subTest('No Imbalanced Warning'): bbox_label_stats = { @@ -370,15 +405,15 @@ def test_check_imbalanced_bbox_dist_in_label(self): } } } - reports = self.validator._check_imbalanced_bbox_dist_in_label( + reports = self.validator._check_imbalanced_dist_in_label( label_name, bbox_label_stats) self.assertTrue(len(reports) == 0) - def test_check_imbalanced_bbox_dist_in_attr(self): + def test_check_imbalanced_dist_in_attr(self): label_name = 'unit' attr_name = 'test' - most = int(self.validator.DEFAULT_BBOX_IMBALANCE * 100) + most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -392,11 +427,11 @@ def test_check_imbalanced_bbox_dist_in_attr(self): } } - reports = self.validator._check_imbalanced_bbox_dist_in_attr( + reports = self.validator._check_imbalanced_dist_in_attr( label_name, attr_name, bbox_attr_stats) self.assertTrue(len(reports) == 1) - self.assertIsInstance(reports[0], ImbalancedBboxDistInAttribute) + self.assertIsInstance(reports[0], ImbalancedDistInAttribute) with self.subTest('No Imbalanced Warning'): bbox_attr_stats = { @@ -409,20 +444,20 @@ def test_check_imbalanced_bbox_dist_in_attr(self): } } - reports = self.validator._check_imbalanced_bbox_dist_in_attr( + reports = self.validator._check_imbalanced_dist_in_attr( label_name, attr_name, bbox_attr_stats) self.assertTrue(len(reports) == 0) def test_check_missing_bbox_annotation(self): stats = { - 'items_missing_bbox': [(1, 'unittest')] + 'items_missing_annotation': [(1, 'unittest')] } - actual_reports = self.validator._check_missing_bbox_annotation(stats) + actual_reports = self.validator._check_missing_annotation(stats) self.assertTrue(len(actual_reports) == 1) - self.assertIsInstance(actual_reports[0], MissingBboxAnnotation) + self.assertIsInstance(actual_reports[0], MissingAnnotation) def test_check_negative_length(self): stats = { @@ -496,14 +531,156 @@ def test_check_far_from_attr_mean(self): self.assertIsInstance(actual_reports[0], FarFromAttrMean) +class TestSegmentationValidator(TestValidatorTemplate): + @classmethod + def setUpClass(cls): + cls.validator = SegmentationValidator() + + def test_check_imbalanced_dist_in_label(self): + label_name = 'unittest' + most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + rest = 100 - most + + with self.subTest('Imbalanced'): + mask_label_stats = { + 'area': { + 'histogram': { + 'counts': [most, rest] + } + } + } + reports = self.validator._check_imbalanced_dist_in_label( + label_name, mask_label_stats) + + self.assertTrue(len(reports) == 1) + self.assertIsInstance(reports[0], ImbalancedDistInLabel) + + with self.subTest('No Imbalanced Warning'): + mask_label_stats = { + 'area': { + 'histogram': { + 'counts': [most - 1, rest] + } + } + } + reports = self.validator._check_imbalanced_dist_in_label( + label_name, mask_label_stats) + + self.assertTrue(len(reports) == 0) + + def test_check_imbalanced_dist_in_attr(self): + label_name = 'unit' + attr_name = 'test' + most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + rest = 100 - most + + with self.subTest('Imbalanced'): + mask_attr_stats = { + 'mock': { + 'x': { + 'histogram': { + 'counts': [most, rest] + } + } + } + } + + reports = self.validator._check_imbalanced_dist_in_attr( + label_name, attr_name, mask_attr_stats) + + self.assertTrue(len(reports) == 1) + self.assertIsInstance(reports[0], ImbalancedDistInAttribute) + + with self.subTest('No Imbalanced Warning'): + mask_attr_stats = { + 'mock': { + 'x': { + 'histogram': { + 'counts': [most - 1, rest] + } + } + } + } + + reports = self.validator._check_imbalanced_dist_in_attr( + label_name, attr_name, mask_attr_stats) + + self.assertTrue(len(reports) == 0) + + def test_check_missing_mask_annotation(self): + stats = { + 'items_missing_annotation': [(1, 'unittest')] + } + + actual_reports = self.validator._check_missing_annotation(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MissingAnnotation) + + def test_check_invalid_value(self): + stats = { + 'items_with_invalid_value': { + ('1', 'unittest'): { + 1: ['x'] + } + } + } + + actual_reports = self.validator._check_invalid_value(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], InvalidValue) + + def test_check_far_from_label_mean(self): + label_name = 'unittest' + mask_label_stats = { + 'w': { + 'items_far_from_mean': { + ('1', 'unittest'): { + 1: 100 + } + }, + 'mean': 0, + } + } + + actual_reports = self.validator._check_far_from_label_mean( + label_name, mask_label_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FarFromLabelMean) + + def test_check_far_from_attr_mean(self): + label_name = 'unit' + attr_name = 'test' + mask_attr_stats = { + 'mock': { + 'w': { + 'items_far_from_mean': { + ('1', 'unittest'): { + 1: 100 + } + }, + 'mean': 0, + } + } + } + + actual_reports = self.validator._check_far_from_attr_mean( + label_name, attr_name, mask_attr_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FarFromAttrMean) + + class TestValidateAnnotations(TestValidatorTemplate): def test_validate_annotations_classification(self): actual_results = validate_annotations(self.dataset, 'classification') with self.subTest('Test of statistics', i=0): actual_stats = actual_results['statistics'] - self.assertEqual(actual_stats['total_label_count'], 8) - self.assertEqual(len(actual_stats['items_missing_label']), 1) + self.assertEqual(actual_stats['total_ann_count'], 8) + self.assertEqual(len(actual_stats['items_missing_annotation']), 1) self.assertEqual(len(actual_stats['items_with_multiple_labels']), 1) label_dist = actual_stats['label_distribution'] @@ -538,7 +715,7 @@ def test_validate_annotations_classification(self): self.assertEqual(report_count_by_type['UndefinedAttribute'], 7) self.assertEqual(report_count_by_type['FewSamplesInAttribute'], 3) self.assertEqual(report_count_by_type['UndefinedLabel'], 2) - self.assertEqual(report_count_by_type['MissingLabelAnnotation'], 1) + self.assertEqual(report_count_by_type['MissingAnnotation'], 1) self.assertEqual(report_count_by_type['MultiLabelAnnotations'], 1) self.assertEqual(report_count_by_type['OnlyOneAttributeValue'], 1) self.assertEqual(report_count_by_type['MissingAttribute'], 1) @@ -557,8 +734,8 @@ def test_validate_annotations_detection(self): with self.subTest('Test of statistics', i=0): actual_stats = actual_results['statistics'] - self.assertEqual(actual_stats['total_bbox_count'], 8) - self.assertEqual(len(actual_stats['items_missing_bbox']), 1) + self.assertEqual(actual_stats['total_ann_count'], 8) + self.assertEqual(len(actual_stats['items_missing_annotation']), 1) self.assertEqual(actual_stats['items_with_negative_length'], {}) self.assertEqual(actual_stats['items_with_invalid_value'], {}) @@ -589,12 +766,12 @@ def test_validate_annotations_detection(self): count_by_type = Counter(report_types) self.assertEqual(len(actual_reports), 45) - self.assertEqual(count_by_type['ImbalancedBboxDistInAttribute'], 32) + self.assertEqual(count_by_type['ImbalancedDistInAttribute'], 32) self.assertEqual(count_by_type['FewSamplesInAttribute'], 4) self.assertEqual(count_by_type['UndefinedAttribute'], 4) - self.assertEqual(count_by_type['ImbalancedBboxDistInLabel'], 2) + self.assertEqual(count_by_type['ImbalancedDistInLabel'], 2) self.assertEqual(count_by_type['UndefinedLabel'], 2) - self.assertEqual(count_by_type['MissingBboxAnnotation'], 1) + self.assertEqual(count_by_type['MissingAnnotation'], 1) with self.subTest('Test of summary', i=2): actual_summary = actual_results['summary'] @@ -605,6 +782,60 @@ def test_validate_annotations_detection(self): self.assertEqual(actual_summary, expected_summary) + def test_validate_annotations_segmentation(self): + actual_results = validate_annotations(self.dataset, 'segmentation') + + with self.subTest('Test of statistics', i=0): + actual_stats = actual_results['statistics'] + self.assertEqual(actual_stats['total_ann_count'], 8) + self.assertEqual(len(actual_stats['items_missing_annotation']), 1) + self.assertEqual(actual_stats['items_with_invalid_value'], {}) + + mask_dist_by_label = actual_stats['mask_distribution_in_label'] + label_prop_stats = mask_dist_by_label['label_1']['area'] + self.assertEqual(label_prop_stats['items_far_from_mean'], {}) + areas = [12, 4, 8] + self.assertEqual(label_prop_stats['mean'], np.mean(areas)) + self.assertEqual(label_prop_stats['stdev'], np.std(areas)) + self.assertEqual(label_prop_stats['min'], np.min(areas)) + self.assertEqual(label_prop_stats['max'], np.max(areas)) + self.assertEqual(label_prop_stats['median'], np.median(areas)) + + mask_dist_by_attr = actual_stats['mask_distribution_in_attribute'] + attr_prop_stats = mask_dist_by_attr['label_0']['a']['1']['area'] + areas = [12, 4] + self.assertEqual(attr_prop_stats['items_far_from_mean'], {}) + self.assertEqual(attr_prop_stats['mean'], np.mean(areas)) + self.assertEqual(attr_prop_stats['stdev'], np.std(areas)) + self.assertEqual(attr_prop_stats['min'], np.min(areas)) + self.assertEqual(attr_prop_stats['max'], np.max(areas)) + self.assertEqual(attr_prop_stats['median'], np.median(areas)) + + mask_dist_item = actual_stats['mask_distribution_in_dataset_item'] + self.assertEqual(sum(mask_dist_item.values()), 8) + + with self.subTest('Test of validation reports', i=1): + actual_reports = actual_results['validation_reports'] + report_types = [r['anomaly_type'] for r in actual_reports] + count_by_type = Counter(report_types) + + self.assertEqual(len(actual_reports), 24) + self.assertEqual(count_by_type['ImbalancedDistInLabel'], 0) + self.assertEqual(count_by_type['ImbalancedDistInAttribute'], 13) + self.assertEqual(count_by_type['MissingAnnotation'], 1) + self.assertEqual(count_by_type['UndefinedLabel'], 2) + self.assertEqual(count_by_type['FewSamplesInAttribute'], 4) + self.assertEqual(count_by_type['UndefinedAttribute'], 4) + + with self.subTest('Test of summary', i=2): + actual_summary = actual_results['summary'] + expected_summary = { + 'errors': 6, + 'warnings': 18 + } + + self.assertEqual(actual_summary, expected_summary) + def test_validate_annotations_invalid_task_type(self): with self.assertRaises(ValueError): validate_annotations(self.dataset, 'INVALID') From 4375cdb8d5eb0068fb281d31e1ae91bfb95decad Mon Sep 17 00:00:00 2001 From: Zoya Maslova Date: Wed, 5 May 2021 14:34:55 +0300 Subject: [PATCH 15/40] Add panoptic and stuff COCO format (#210) * add coco stuff and panoptic formats * update CHANGELOG Co-authored-by: Maxim Zhiltsov --- CHANGELOG.md | 1 + README.md | 2 +- datumaro/plugins/coco_format/converter.py | 83 ++++++++++++ datumaro/plugins/coco_format/extractor.py | 97 ++++++++++++-- datumaro/plugins/coco_format/format.py | 5 +- datumaro/plugins/coco_format/importer.py | 5 +- datumaro/plugins/datumaro_format/converter.py | 7 +- datumaro/util/mask_tools.py | 13 +- docs/user_manual.md | 2 +- .../annotations/panoptic_val.json | 75 +++++++++++ .../annotations/panoptic_val/000000000001.png | Bin 0 -> 78 bytes .../coco_panoptic/images/val/000000000001.jpg | Bin 0 -> 631 bytes .../coco_stuff/annotations/stuff_val.json | 50 ++++++++ .../coco_stuff/images/val/000000000001.jpg | Bin 0 -> 631 bytes tests/test_coco_format.py | 118 +++++++++++++++++- 15 files changed, 439 insertions(+), 19 deletions(-) create mode 100644 tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val.json create mode 100644 tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val/000000000001.png create mode 100644 tests/assets/coco_dataset/coco_panoptic/images/val/000000000001.jpg create mode 100644 tests/assets/coco_dataset/coco_stuff/annotations/stuff_val.json create mode 100644 tests/assets/coco_dataset/coco_stuff/images/val/000000000001.jpg diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1be7bcde..c5b775ee01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for escaping in attribiute values in LabelMe format () - Support for Segmentation Splitting () - Support for CIFAR-10/100 dataset format () +- Support COCO panoptic and stuff format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/README.md b/README.md index 4293d400bc..2c3a2f70ba 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ CVAT annotations ---> Publication, statistics etc. [(Back to top)](#table-of-contents) - Dataset reading, writing, conversion in any direction. [Supported formats](docs/user_manual.md#supported-formats): - - [COCO](http://cocodataset.org/#format-data) (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`*) + - [COCO](http://cocodataset.org/#format-data) (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`, `panoptic`, `stuff`) - [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/htmldoc/index.html) (`classification`, `detection`, `segmentation`, `action_classification`, `person_layout`) - [YOLO](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) (`bboxes`) - [TF Detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/using_your_own_dataset.md) (`bboxes`, `masks`) diff --git a/datumaro/plugins/coco_format/converter.py b/datumaro/plugins/coco_format/converter.py index 0caf89de3b..36781a0396 100644 --- a/datumaro/plugins/coco_format/converter.py +++ b/datumaro/plugins/coco_format/converter.py @@ -5,6 +5,7 @@ import json import logging as log +import numpy as np import os import os.path as osp from enum import Enum @@ -19,6 +20,7 @@ _COORDINATE_ROUNDING_DIGITS, AnnotationType, Points) from datumaro.components.dataset import ItemStatus from datumaro.util import cast, find, str_to_bool +from datumaro.util.image import save_image from .format import CocoPath, CocoTask @@ -451,6 +453,67 @@ def save_annotations(self, item): self.annotations.append(elem) +class _StuffConverter(_InstancesConverter): + pass + +class _PanopticConverter(_TaskConverter): + def write(self, path): + with open(path, 'w') as outfile: + json.dump(self._data, outfile) + + def save_categories(self, dataset): + label_categories = dataset.categories().get(AnnotationType.label) + if label_categories is None: + return + + for idx, cat in enumerate(label_categories.items): + self.categories.append({ + 'id': 1 + idx, + 'name': cast(cat.name, str, ''), + 'supercategory': cast(cat.parent, str, ''), + 'isthing': 0, # TODO: can't represent this information yet + }) + + def save_annotations(self, item): + if not item.has_image: + return + + ann_filename = item.id + CocoPath.PANOPTIC_EXT + + segments_info = list() + masks = [] + next_id = self._min_ann_id + for ann in item.annotations: + if ann.type != AnnotationType.mask: + continue + + if not ann.id: + ann.id = next_id + next_id += 1 + + segment_info = {} + segment_info['id'] = ann.id + segment_info['category_id'] = cast(ann.label, int, -1) + 1 + segment_info['area'] = float(ann.get_area()) + segment_info['bbox'] = [float(p) for p in ann.get_bbox()] + segment_info['iscrowd'] = cast(ann.attributes.get("is_crowd"), int, 0) + segments_info.append(segment_info) + masks.append(ann) + + if masks: + pan_format = mask_tools.merge_masks( + ((m.image, m.id) for m in masks), + start=np.zeros(item.image.size, dtype=np.uint32)) + save_image(osp.join(self._context._segmentation_dir, ann_filename), + mask_tools.index2bgr(pan_format), create_dir=True) + + elem = { + 'image_id': self._get_image_id(item), + 'file_name': ann_filename, + 'segments_info': segments_info + } + self.annotations.append(elem) + class CocoConverter(Converter): @staticmethod def _split_tasks_string(s): @@ -497,6 +560,8 @@ def build_cmdline_parser(cls, **kwargs): CocoTask.person_keypoints: _KeypointsConverter, CocoTask.captions: _CaptionsConverter, CocoTask.labels: _LabelsConverter, + CocoTask.panoptic: _PanopticConverter, + CocoTask.stuff: _StuffConverter, } def __init__(self, extractor, save_dir, @@ -541,6 +606,11 @@ def _make_dirs(self): self._ann_dir = osp.join(self._save_dir, CocoPath.ANNOTATIONS_DIR) os.makedirs(self._ann_dir, exist_ok=True) + def _make_segmentation_dir(self, subset_name): + self._segmentation_dir = osp.join(self._save_dir, + CocoPath.ANNOTATIONS_DIR, 'panoptic_'+ subset_name) + os.makedirs(self._segmentation_dir, exist_ok=True) + def _make_task_converter(self, task): if task not in self._TASK_CONVERTER: raise NotImplementedError() @@ -568,6 +638,8 @@ def apply(self): task_converters = self._make_task_converters() for task_conv in task_converters.values(): task_conv.save_categories(subset) + if CocoTask.panoptic in task_converters: + self._make_segmentation_dir(subset_name) for item in subset: if self._save_images: @@ -637,3 +709,14 @@ class CocoLabelsConverter(CocoConverter): def __init__(self, *args, **kwargs): kwargs['tasks'] = CocoTask.labels super().__init__(*args, **kwargs) + +class CocoPanopticConverter(CocoConverter): + def __init__(self, *args, **kwargs): + kwargs['tasks'] = CocoTask.panoptic + super().__init__(*args, **kwargs) + +class CocoStuffConverter(CocoConverter): + def __init__(self, *args, **kwargs): + kwargs['tasks'] = CocoTask.stuff + kwargs['segmentation_mode'] = SegmentationMode.mask + super().__init__(*args, **kwargs) diff --git a/datumaro/plugins/coco_format/extractor.py b/datumaro/plugins/coco_format/extractor.py index 29b97f7e27..faecf79f8e 100644 --- a/datumaro/plugins/coco_format/extractor.py +++ b/datumaro/plugins/coco_format/extractor.py @@ -4,18 +4,20 @@ # SPDX-License-Identifier: MIT from collections import OrderedDict +import json import logging as log import os.path as osp from pycocotools.coco import COCO import pycocotools.mask as mask_utils -from datumaro.components.extractor import (SourceExtractor, +from datumaro.components.extractor import (CompiledMask, Mask, SourceExtractor, DEFAULT_SUBSET_NAME, DatasetItem, AnnotationType, Label, RleMask, Points, Polygon, Bbox, Caption, LabelCategories, PointsCategories ) -from datumaro.util.image import Image +from datumaro.util.image import Image, lazy_image, load_image +from datumaro.util.mask_tools import bgr2index from .format import CocoTask, CocoPath @@ -42,16 +44,24 @@ def __init__(self, path, task, merge_instance_polygons=False, subset=None): self._merge_instance_polygons = merge_instance_polygons - loader = self._make_subset_loader(path) - self._load_categories(loader) - self._items = list(self._load_items(loader).values()) + if self._task == CocoTask.panoptic: + #panoptic is not added to pycocotools + panoptic_config = self._load_panoptic_config(path) + panoptic_images = osp.splitext(path)[0] + + self._load_panoptic_categories(panoptic_config) + self._items = list(self._load_panoptic_items(panoptic_config, + panoptic_images).values()) + else: + loader = self._make_subset_loader(path) + self._load_categories(loader) + self._items = list(self._load_items(loader).values()) @staticmethod def _make_subset_loader(path): # COCO API has an 'unclosed file' warning coco_api = COCO() with open(path, 'r') as f: - import json dataset = json.load(f) coco_api.dataset = dataset @@ -62,9 +72,7 @@ def _load_categories(self, loader): self._categories = {} if self._task in [CocoTask.instances, CocoTask.labels, - CocoTask.person_keypoints, - # TODO: Task.stuff, CocoTask.panoptic - ]: + CocoTask.person_keypoints, CocoTask.stuff]: label_categories, label_map = self._load_label_categories(loader) self._categories[AnnotationType.label] = label_categories self._label_map = label_map @@ -100,6 +108,22 @@ def _load_person_kp_categories(self, loader): return categories + @staticmethod + def _load_panoptic_config(path): + with open(path, 'r') as f: + return json.load(f) + + def _load_panoptic_categories(self, config): + label_categories = LabelCategories() + label_map = {} + for idx, cat in enumerate(config['categories']): + label_map[cat['id']] = idx + label_categories.add(name=cat['name'], + parent=cat.get('supercategory')) + + self._categories[AnnotationType.label] = label_categories + self._label_map = label_map + def _load_items(self, loader): items = OrderedDict() @@ -124,6 +148,48 @@ def _load_items(self, loader): return items + def _load_panoptic_items(self, config, panoptic_images): + items = OrderedDict() + + imgs_info = {} + for img in config['images']: + imgs_info[img['id']] = img + + for ann in config['annotations']: + img_id = int(ann['image_id']) + image_path = osp.join(self._images_dir, imgs_info[img_id]['file_name']) + image_size = (imgs_info[img_id].get('height'), + imgs_info[img_id].get('width')) + if all(image_size): + image_size = (int(image_size[0]), int(image_size[1])) + else: + image_size = None + image = Image(path=image_path, size=image_size) + anns = [] + + mask_path = osp.join(panoptic_images, ann['file_name']) + mask = lazy_image(mask_path, loader=self._load_pan_mask) + mask = CompiledMask(instance_mask=mask) + for segm_info in ann['segments_info']: + cat_id = self._get_label_id(segm_info) + segm_id = segm_info['id'] + attributes = { 'is_crowd': bool(segm_info['iscrowd']) } + anns.append(Mask(image=mask.lazy_extract(segm_id), + label=cat_id, id=segm_id, + group=segm_id, attributes=attributes)) + + items[img_id] = DatasetItem( + id=osp.splitext(imgs_info[img_id]['file_name'])[0], + subset=self._subset, image=image, + annotations=anns, attributes={'id': img_id}) + return items + + @staticmethod + def _load_pan_mask(path): + mask = load_image(path) + mask = bgr2index(mask) + return mask + def _get_label_id(self, ann): cat_id = ann.get('category_id') if cat_id in [0, None]: @@ -147,7 +213,8 @@ def _load_annotations(self, ann, image_info=None): group = ann_id # make sure all tasks' annotations are merged - if self._task in [CocoTask.instances, CocoTask.person_keypoints]: + if self._task in [CocoTask.instances, CocoTask.person_keypoints, + CocoTask.stuff]: x, y, w, h = ann['bbox'] label_id = self._get_label_id(ann) @@ -250,3 +317,13 @@ class CocoLabelsExtractor(_CocoExtractor): def __init__(self, path, **kwargs): kwargs['task'] = CocoTask.labels super().__init__(path, **kwargs) + +class CocoPanopticExtractor(_CocoExtractor): + def __init__(self, path, **kwargs): + kwargs['task'] = CocoTask.panoptic + super().__init__(path, **kwargs) + +class CocoStuffExtractor(_CocoExtractor): + def __init__(self, path, **kwargs): + kwargs['task'] = CocoTask.stuff + super().__init__(path, **kwargs) diff --git a/datumaro/plugins/coco_format/format.py b/datumaro/plugins/coco_format/format.py index 5129d49d9a..7a37bb709c 100644 --- a/datumaro/plugins/coco_format/format.py +++ b/datumaro/plugins/coco_format/format.py @@ -12,8 +12,8 @@ 'captions', 'labels', # extension, does not exist in the original COCO format 'image_info', - # 'panoptic', - # 'stuff', + 'panoptic', + 'stuff', ]) class CocoPath: @@ -21,3 +21,4 @@ class CocoPath: ANNOTATIONS_DIR = 'annotations' IMAGE_EXT = '.jpg' + PANOPTIC_EXT = '.png' diff --git a/datumaro/plugins/coco_format/importer.py b/datumaro/plugins/coco_format/importer.py index f613143e15..2e8f8a2ac8 100644 --- a/datumaro/plugins/coco_format/importer.py +++ b/datumaro/plugins/coco_format/importer.py @@ -21,6 +21,8 @@ class CocoImporter(Importer): CocoTask.captions: 'coco_captions', CocoTask.labels: 'coco_labels', CocoTask.image_info: 'coco_image_info', + CocoTask.panoptic: 'coco_panoptic', + CocoTask.stuff: 'coco_stuff', } @classmethod @@ -39,7 +41,8 @@ def __call__(self, path, **extra_params): # TODO: should be removed when proper label merging is implemented conflicting_types = {CocoTask.instances, - CocoTask.person_keypoints, CocoTask.labels} + CocoTask.person_keypoints, CocoTask.labels, + CocoTask.panoptic, CocoTask.stuff} ann_types = set(t for s in subsets.values() for t in s) \ & conflicting_types if 1 <= len(ann_types): diff --git a/datumaro/plugins/datumaro_format/converter.py b/datumaro/plugins/datumaro_format/converter.py index 6e9de7142c..18d16c1428 100644 --- a/datumaro/plugins/datumaro_format/converter.py +++ b/datumaro/plugins/datumaro_format/converter.py @@ -131,11 +131,16 @@ def _convert_mask_object(self, obj): rle = mask_utils.encode( np.require(obj.image, dtype=np.uint8, requirements='F')) + if isinstance(rle['counts'], str): + counts = rle['counts'] + else: + counts = rle['counts'].decode('ascii') + converted.update({ 'label_id': cast(obj.label, int), 'rle': { # serialize as compressed COCO mask - 'counts': rle['counts'].decode('ascii'), + 'counts': counts, 'size': list(int(c) for c in rle['size']), }, 'z_order': obj.z_order, diff --git a/datumaro/util/mask_tools.py b/datumaro/util/mask_tools.py index b6c2bc9462..bd763dffea 100644 --- a/datumaro/util/mask_tools.py +++ b/datumaro/util/mask_tools.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: MIT +from itertools import chain import numpy as np from datumaro.util.image import lazy_image, load_image @@ -112,6 +113,13 @@ def make_binary_mask(mask): return mask return mask.astype(bool) +def bgr2index(img): + if img.dtype.kind not in {'b', 'i', 'u'}: + img = img.astype(np.uint8) + return (img[..., 0] << 16) + (img[..., 1] << 8) + img[..., 2] + +def index2bgr(id_map): + return np.dstack((id_map >> 16, id_map >> 8, id_map)).astype(np.uint8) def load_mask(path, inverse_colormap=None): mask = load_image(path, dtype=np.uint8) @@ -279,7 +287,7 @@ def find_mask_bbox(mask): y0, y1 = np.where(rows)[0][[0, -1]] return [x0, y0, x1 - x0, y1 - y0] -def merge_masks(masks): +def merge_masks(masks, start=None): """ Merges masks into one, mask order is responsible for z order. To avoid memory explosion on mask materialization, consider passing @@ -288,6 +296,9 @@ def merge_masks(masks): Inputs: a sequence of index masks or (binary mask, index) pairs Outputs: an index mask """ + if start is not None: + masks = chain([start], masks) + it = iter(masks) try: diff --git a/docs/user_manual.md b/docs/user_manual.md index 9930e31394..a1602be2ec 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -84,7 +84,7 @@ import datumaro ## Supported Formats List of supported formats: -- MS COCO (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`*) +- MS COCO (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`, `panoptic`, `stuff`) - [Format specification](http://cocodataset.org/#format-data) - [Dataset example](../tests/assets/coco_dataset) - `labels` are our extension - like `instances` with only `category_id` diff --git a/tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val.json b/tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val.json new file mode 100644 index 0000000000..c945de7ca5 --- /dev/null +++ b/tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val.json @@ -0,0 +1,75 @@ +{ + "licenses": [{ + "name": "", + "id": 0, + "url": "" + }], + "info": { + "contributor": "", + "date_created": "", + "description": "", + "url": "", + "version": "", + "year": "" + }, + "categories": [ + { + "id": 1, + "name": "a", + "supercategory": "", + "isthing": 1 + }, + { + "id": 2, + "name": "b", + "supercategory": "", + "isthing": 1 + }, + { + "id": 3, + "name": "c", + "supercategory": "", + "isthing": 1 + }, + { + "id": 4, + "name": "d", + "supercategory": "", + "isthing": 1 + } + ], + "images": [ + { + "id": 40, + "width": 5, + "height": 1, + "file_name": "000000000001.jpg", + "license": 0, + "flickr_url": "", + "coco_url": "", + "date_captured": 0 + } + ], + "annotations": [ + { + "image_id": 40, + "file_name": "000000000001.png", + "segments_info": [ + { + "id": 7, + "category_id": 4, + "area": 2.0, + "bbox": [2.0, 0.0, 1.0, 0.0], + "iscrowd": 0 + }, + { + "id": 20, + "category_id": 2, + "area": 2.0, + "bbox": [1.0, 0.0, 3.0, 0.0], + "iscrowd": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val/000000000001.png b/tests/assets/coco_dataset/coco_panoptic/annotations/panoptic_val/000000000001.png new file mode 100644 index 0000000000000000000000000000000000000000..e471bfed416252e6619cfb903be67ce3e1104417 GIT binary patch literal 78 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1j!2~2{&iT9qNQrv7IEHY@CZ{AM2qY%_IK#lS bhlhuO^AMw0yP>lwPyvIdtDnm{r-UW|NK_FF literal 0 HcmV?d00001 diff --git a/tests/assets/coco_dataset/coco_panoptic/images/val/000000000001.jpg b/tests/assets/coco_dataset/coco_panoptic/images/val/000000000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a082a80324c398d11403c8aba2946f58746be4ea GIT binary patch literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<5A1R0qH8UG()kO#Vxl@SaWpn#EynT3^&or9B$8>nEb z00R>vGcywlGb<|#3s7|}P@aKBkX1<0(2-3zFp*uUP{gQl;zAB(r;P_igD!qhF-|IK z;^Yz&myncFRa4i{)G{$OGqmaka3YSZQ|TeofBv2)j3M&~ka)>xhT)6Qdr?PR-2hpUWi(FzVCJ$9Vg1iRy l8F3zKBFkrRk0JbZi-Cuk5g2*Qf(-TyAGkCYHQ4{Z2>|I(&5!^9 literal 0 HcmV?d00001 diff --git a/tests/assets/coco_dataset/coco_stuff/annotations/stuff_val.json b/tests/assets/coco_dataset/coco_stuff/annotations/stuff_val.json new file mode 100644 index 0000000000..51a654f9d1 --- /dev/null +++ b/tests/assets/coco_dataset/coco_stuff/annotations/stuff_val.json @@ -0,0 +1,50 @@ +{ + "licenses": [ + { + "name": "", + "id": 0, + "url": "" + } + ], + "info": { + "contributor": "", + "date_created": "", + "description": "", + "url": "", + "version": "", + "year": "" + }, + "categories": [ + { + "id": 1, + "name": "TEST", + "supercategory": "" + } + ], + "images": [ + { + "id": 1, + "width": 5, + "height": 10, + "file_name": "000000000001.jpg", + "license": 0, + "flickr_url": "", + "coco_url": "", + "date_captured": 0 + } + ], + "annotations": [ + { + "id": 2, + "image_id": 1, + "category_id": 1, + "segmentation": { + "counts": [0, 10, 5, 5, 5, 5, 0, 10, 10, 0], + "size": [10, 5] + }, + "area": 30, + "bbox": [0, 0, 10, 4], + "iscrowd": 0 + } + ] + } diff --git a/tests/assets/coco_dataset/coco_stuff/images/val/000000000001.jpg b/tests/assets/coco_dataset/coco_stuff/images/val/000000000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8bce84d3bf50bd756621338e0da944a42428fb06 GIT binary patch literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf} Date: Mon, 10 May 2021 16:42:31 +0900 Subject: [PATCH 16/40] update detection splitter algorithm from # of samples to # of instances (#235) --- datumaro/plugins/splitter.py | 118 ++++++++++++++++++----------------- docs/user_manual.md | 10 ++- tests/test_splitter.py | 49 ++++++++------- 3 files changed, 91 insertions(+), 86 deletions(-) diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index abc391ab19..786828b3c5 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -4,6 +4,7 @@ import logging as log import numpy as np +import copy from math import gcd from enum import Enum @@ -295,7 +296,7 @@ def _get_sections(dataset_size, ratio): def _group_by_attr(items): """ Args: - items: list of (idx, ann). ann is the annotation from Label object. + items: list of (idx_img, ann). ann is the annotation from Label object. Returns: by_attributes: dict of { combination-of-attrs : list of index } """ @@ -315,17 +316,17 @@ def _is_float(value): # group by attributes by_attributes = dict() - for idx, ann in items: + for idx_img, ann in items: # ignore numeric attributes filtered = {} - for k, v in ann.attributes.items(): - if _is_float(v): + for attr, value in ann.attributes.items(): + if _is_float(value): continue - filtered[k] = v + filtered[attr] = value attributes = tuple(sorted(filtered.items())) if attributes not in by_attributes: by_attributes[attributes] = [] - by_attributes[attributes].append(idx) + by_attributes[attributes].append(idx_img) return by_attributes @@ -344,9 +345,9 @@ def _split_indice(indice): for _, items in datasets.items(): np.random.shuffle(items) by_attributes = self._group_by_attr(items) - attr_names = list(by_attributes.keys()) - np.random.shuffle(attr_names) # add randomness - for attr in attr_names: + attr_combinations = list(by_attributes.keys()) + np.random.shuffle(attr_combinations) # add randomness + for attr in attr_combinations: indice = by_attributes[attr] quo = len(indice) // required if quo > 0: @@ -719,17 +720,19 @@ def __init__(self, dataset, splits, task, seed=None): def _group_by_labels(self, dataset): by_labels = dict() unlabeled = [] + for idx, item in enumerate(dataset): - bbox_anns = [a for a in item.annotations if a.type in self.annotation_type] - if len(bbox_anns) == 0: + instance_anns = [a for a in item.annotations if a.type in self.annotation_type] + if len(instance_anns) == 0: unlabeled.append(idx) continue - for ann in bbox_anns: - label = getattr(ann, "label", None) + for instance_ann in instance_anns: + label = getattr(instance_ann, "label", None) if label not in by_labels: - by_labels[label] = [(idx, ann)] + by_labels[label] = [(idx, instance_ann)] else: - by_labels[label].append((idx, ann)) + by_labels[label].append((idx, instance_ann)) + return by_labels, unlabeled def _split_dataset(self): @@ -746,79 +749,80 @@ def _split_dataset(self): for _, items in by_labels.items(): by_attributes = self._group_by_attr(items) # merge groups which have too small samples. - attr_names = list(by_attributes.keys()) - np.random.shuffle(attr_names) # add randomless + attr_combinations = list(by_attributes.keys()) + np.random.shuffle(attr_combinations) # add randomless cluster = [] - minumum = max(required, len(items) * 0.1) # temp solution - for attr in attr_names: + min_cluster = max(required, len(items) * 0.01) # temp solution + for attr in attr_combinations: indice = by_attributes[attr] - if len(indice) >= minumum: + if len(indice) >= min_cluster: by_combinations.append(indice) else: cluster.extend(indice) - if len(cluster) >= minumum: + if len(cluster) >= min_cluster: by_combinations.append(cluster) cluster = [] + if len(cluster) > 0: by_combinations.append(cluster) cluster = [] total = len(self._extractor) - # total number of GT samples per label-attr combinations n_combs = [len(v) for v in by_combinations] # 3-1. initially count per-image GT samples counts_all = {} - for idx in range(total): - if idx not in unlabeled: - counts_all[idx] = dict() + for idx_img in range(total): + if idx_img not in unlabeled: + counts_all[idx_img] = dict() for idx_comb, indice in enumerate(by_combinations): - for idx in indice: - if idx_comb not in counts_all[idx]: - counts_all[idx] = {idx_comb: 1} + for idx_img in indice: + if idx_comb not in counts_all[idx_img]: + counts_all[idx_img][idx_comb] = 1 else: - counts_all[idx][idx_comb] += 1 - - init_scores = {} - for idx, counts in counts_all.items(): - norm_sum = 0.0 - for idx_comb, count in counts.items(): - norm_sum += count / n_combs[idx_comb] - init_scores[idx] = norm_sum + counts_all[idx_img][idx_comb] += 1 by_splits = dict() for sname in self._subsets: by_splits[sname] = [] - target_size = dict() - expected = [] # expected numbers of per split GT samples + target_ins = [] # target instance numbers to be split for sname, ratio in zip(subsets, sratio): - target_size[sname] = (total - len(unlabeled)) * ratio - expected.append([sname, np.array(n_combs) * ratio]) + target_ins.append([sname, np.array(n_combs) * ratio]) + + init_scores = {} + for idx_img, distributions in counts_all.items(): + norm_sum = 0.0 + for idx_comb, dis in distributions.items(): + norm_sum += dis / n_combs[idx_comb] + init_scores[idx_img] = norm_sum - # functions for keep the # of annotations not exceed the expected num + by_scores = dict() + for idx_img, score in init_scores.items(): + if score not in by_scores: + by_scores[score] = [idx_img] + else: + by_scores[score].append(idx_img) + + # functions for keep the # of annotations not exceed the target_ins num def compute_penalty(counts, n_combs): p = 0 for idx_comb, v in counts.items(): - p += max(0, (v / n_combs[idx_comb]) - 1.0) + if n_combs[idx_comb] <= 0: + p += 1 + else: + p += max(0, (v / n_combs[idx_comb]) - 1.0) + return p def update_nc(counts, n_combs): for idx_comb, v in counts.items(): - n_combs[idx_comb] = max(0, n_combs[idx_comb] - v) - if n_combs[idx_comb] == 0: - n_combs[idx_comb] = -1 - - by_scores = dict() - for idx, score in init_scores.items(): - if score not in by_scores: - by_scores[score] = [idx] - else: - by_scores[score].append(idx) + n_combs[idx_comb] = n_combs[idx_comb] - v # 3-2. assign each DatasetItem to a split, one by one + actual_ins = copy.deepcopy(target_ins) for score in sorted(by_scores.keys(), reverse=True): indice = by_scores[score] np.random.shuffle(indice) # add randomness for the same score @@ -827,12 +831,12 @@ def update_nc(counts, n_combs): counts = counts_all[idx] # shuffling split order to add randomness # when two or more splits have the same penalty value - np.random.shuffle(expected) + np.random.shuffle(actual_ins) pp = [] - for sname, nc in expected: - if target_size[sname] <= len(by_splits[sname]): - # the split has enough images, + for sname, nc in actual_ins: + if np.sum(nc) <= 0: + # the split has enough instances, # stop adding more images to this split pp.append(1e08) else: @@ -842,7 +846,7 @@ def update_nc(counts, n_combs): # we push an image to a split with the minimum penalty midx = np.argmin(pp) - sname, nc = expected[midx] + sname, nc = actual_ins[midx] by_splits[sname].append(idx) update_nc(counts, nc) diff --git a/docs/user_manual.md b/docs/user_manual.md index a1602be2ec..df9bb33124 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1037,8 +1037,8 @@ Example: split a dataset randomly to `train` and `test` subsets, ratio is 2:1 datum transform -t random_split -- --subset train:.67 --subset test:.33 ``` -Example: split a dataset in task-specific manner. Supported tasks are -classification, detection, re-identification and segmentation. +Example: split a dataset in task-specific manner. The tasks supported are +classification, detection, segmentation and re-identification. ``` bash datum transform -t split -- \ @@ -1081,9 +1081,7 @@ datum transform -t rename -- -e '|pattern|replacement|' datum transform -t rename -- -e '|frame_(\d+)|\\1|' ``` -Example: Sampling dataset items, subset `train` is divided into `sampled`(sampled_subset) and `unsampled` -- `train` has 100 data, and 20 samples are selected. There are `sampled`(20 samples) and 80 `unsampled`(80 datas) subsets. -- Remove `train` subset (if sampled_subset=`train` or unsampled_name=`train`, still remain) +Example: sampling dataset items as many as the number of target samples with sampling method entered by the user, divide into `sampled` and `unsampled` subsets - There are five methods of sampling the m option. - `topk`: Return the k with high uncertainty data - `lowk`: Return the k with low uncertainty data @@ -1101,7 +1099,7 @@ datum transform -t sampler -- \ -k 20 ``` -Example : Control number of outputs to 100 after NDR +Example : control number of outputs to 100 after NDR - There are two methods in NDR e option - `random`: sample from removed data randomly - `similarity`: sample from removed data with ascending diff --git a/tests/test_splitter.py b/tests/test_splitter.py index 4c233f0eb2..a6b778e192 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -79,7 +79,7 @@ def test_split_for_classification_multi_class_no_attr(self): task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(42, len(actual.get_subset("train"))) self.assertEqual(18, len(actual.get_subset("test"))) @@ -105,7 +105,7 @@ def test_split_for_classification_single_class_single_attr(self): task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(42, len(actual.get_subset("train"))) self.assertEqual(18, len(actual.get_subset("test"))) @@ -140,7 +140,7 @@ def test_split_for_classification_single_class_multi_attr(self): with self.subTest("zero remainder"): splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(84, len(actual.get_subset("train"))) self.assertEqual(36, len(actual.get_subset("test"))) @@ -165,7 +165,7 @@ def test_split_for_classification_single_class_multi_attr(self): with self.subTest("non-zero remainder"): splits = [("train", 0.95), ("test", 0.05)] - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(114, len(actual.get_subset("train"))) self.assertEqual(6, len(actual.get_subset("test"))) @@ -189,7 +189,7 @@ def test_split_for_classification_multi_label_with_attr(self): task = splitter.SplitTask.classification.name splits = [("train", 0.7), ("test", 0.3)] - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) train = actual.get_subset("train") test = actual.get_subset("test") @@ -243,7 +243,7 @@ def test_split_for_classification_zero_ratio(self): splits = [("train", 0.1), ("val", 0.9), ("test", 0.0)] task = splitter.SplitTask.classification.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(1, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) @@ -255,7 +255,7 @@ def test_split_for_classification_unlabeled(self): source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.7), ("test", 0.3)] task = splitter.SplitTask.classification.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(7, len(actual.get_subset("train"))) self.assertEqual(3, len(actual.get_subset("test"))) @@ -266,7 +266,7 @@ def test_split_for_classification_unlabeled(self): source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.7), ("test", 0.3)] task = splitter.SplitTask.classification.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(7, len(actual.get_subset("train"))) self.assertEqual(3, len(actual.get_subset("test"))) @@ -405,7 +405,7 @@ def test_split_for_reidentification_rebalance(self): task = splitter.SplitTask.reid.name splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] query = 0.4 / 0.7 - actual = splitter.Split(source, task, splits, query) + actual = splitter.Split(source, task, splits, query, seed=100) self.assertEqual(350, len(actual.get_subset("train"))) self.assertEqual(140, len(actual.get_subset("val"))) @@ -420,7 +420,7 @@ def test_split_for_reidentification_unlabeled(self): iterable = [DatasetItem(i, annotations=[]) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.6), ("test", 0.4)] - actual = splitter.Split(source, task, splits, query) + actual = splitter.Split(source, task, splits, query, seed=100) self.assertEqual(10, len(actual.get_subset("not-supported"))) with self.subTest("multi label"): @@ -428,7 +428,7 @@ def test_split_for_reidentification_unlabeled(self): iterable = [DatasetItem(i, annotations=anns) for i in range(10)] source = Dataset.from_iterable(iterable, categories=["a", "b"]) splits = [("train", 0.6), ("test", 0.4)] - actual = splitter.Split(source, task, splits, query) + actual = splitter.Split(source, task, splits, query, seed=100) self.assertEqual(10, len(actual.get_subset("not-supported"))) @@ -827,7 +827,7 @@ def test_split_for_detection(self): test=test, task=task, ): - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(train, len(actual.get_subset("train"))) self.assertEqual(val, len(actual.get_subset("val"))) @@ -858,7 +858,7 @@ def test_split_for_detection_with_unlabeled(self): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] task = splitter.SplitTask.detection.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(10, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) @@ -898,7 +898,7 @@ def test_no_subset_name_and_count_restriction(self): config = {"label1": {"attrs": None, "counts": 10}} task = splitter.SplitTask.classification.name source = self._generate_dataset(config) - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(5, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) self.assertEqual(1, len(actual.get_subset("valid2"))) @@ -912,10 +912,10 @@ def test_no_subset_name_and_count_restriction(self): nimages=10, ) task = splitter.SplitTask.detection.name - actual = splitter.Split(source, task, splits) - self.assertEqual(5, len(actual.get_subset("_train"))) + actual = splitter.Split(source, task, splits, seed=21) + self.assertEqual(4, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) - self.assertEqual(1, len(actual.get_subset("valid2"))) + self.assertEqual(2, len(actual.get_subset("valid2"))) self.assertEqual(2, len(actual.get_subset("test*"))) self.assertEqual(1, len(actual.get_subset("test2"))) @@ -926,7 +926,7 @@ def test_no_subset_name_and_count_restriction(self): nimages=10, ) task = splitter.SplitTask.detection.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(5, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) self.assertEqual(1, len(actual.get_subset("valid2"))) @@ -938,7 +938,7 @@ def test_no_subset_name_and_count_restriction(self): with_attr=True, nimages=10, ) - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(5, len(actual.get_subset("_train"))) self.assertEqual(1, len(actual.get_subset("valid"))) self.assertEqual(1, len(actual.get_subset("valid2"))) @@ -977,7 +977,7 @@ def test_split_for_segmentation(self): test=test, task=task, ): - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(train, len(actual.get_subset("train"))) self.assertEqual(val, len(actual.get_subset("val"))) @@ -1008,6 +1008,7 @@ def test_split_for_segmentation(self): params.append((dtype, with_attr, 10, 5, 3, 2)) params.append((dtype, with_attr, 10, 7, 0, 3)) + expected = [] for dtype, with_attr, nimages, train, val, test in params: source, _ = self._generate_detection_segmentation_dataset( annotation_type=self._get_append_polygon(dtype), @@ -1029,7 +1030,9 @@ def test_split_for_segmentation(self): test=test, task=task, ): - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=21) + + expected.append([dtype, with_attr, len(actual.get_subset("train")), len(actual.get_subset("val")), len(actual.get_subset("test"))]) self.assertEqual(train, len(actual.get_subset("train"))) self.assertEqual(val, len(actual.get_subset("val"))) @@ -1064,7 +1067,7 @@ def test_split_for_segmentation_with_unlabeled(self): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] task = splitter.SplitTask.segmentation.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(10, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) @@ -1080,7 +1083,7 @@ def test_split_for_segmentation_with_unlabeled(self): splits = [("train", 0.5), ("val", 0.2), ("test", 0.3)] task = splitter.SplitTask.segmentation.name - actual = splitter.Split(source, task, splits) + actual = splitter.Split(source, task, splits, seed=100) self.assertEqual(10, len(actual.get_subset("train"))) self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) From d8cb65db8fb3646ea45127560ec3bd8118ac0cb1 Mon Sep 17 00:00:00 2001 From: Jihyeon Yi Date: Mon, 10 May 2021 16:45:51 +0900 Subject: [PATCH 17/40] add documentation for validator (#233) * add documentation for validator --- README.md | 1 + docs/user_manual.md | 150 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/README.md b/README.md index 2c3a2f70ba..00697b4bbe 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ CVAT annotations ---> Publication, statistics etc. - Simple checking for errors - Comparison with model infernece - Merging and comparison of multiple datasets + - Annotation validation based on the task type(classification, etc) - Dataset comparison - Dataset statistics (image mean and std, annotation statistics) - Model integration diff --git a/docs/user_manual.md b/docs/user_manual.md index df9bb33124..cee40321dc 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -20,6 +20,7 @@ - [Compare projects](#compare-projects) - [Obtaining project info](#get-project-info) - [Obtaining project statistics](#get-project-statistics) + - [Validate project annotations](#validate-project-annotations) - [Register model](#register-model) - [Run inference](#run-model) - [Run inference explanation](#explain-inference) @@ -878,6 +879,155 @@ datum stats -p test_project + +### Validate project annotations + +This command inspects annotations with respect to the task type +and stores the result in JSON file. + +The task types supported are `classification`, `detection`, and `segmentation`. + +The validation result contains +- annotation statistics based on the task type +- validation reports, such as + - items not having annotations + - items having undefined annotations + - imbalanced distribution in class/attributes + - too small or large values +- summary + +Usage: + +``` bash +datum validate --help + +datum validate -p +``` + +Validation Result: + +
+ +``` bash +{ + 'statistics': { + ## common statistics + 'label_distribution': { + 'defined_labels': , # : + 'undefined_labels': + # : { + # 'count': , + # 'items_with_undefined_label': [, ] + # } + }, + 'attribute_distribution': { + 'defined_attributes': , + # : { + # : { + # 'distribution': {: , }, + # 'items_missing_attribute': [, ] + # } + # } + 'undefined_attributes': + # : { + # : { + # 'distribution': {: , }, + # 'items_with_undefined_attr': [, ] + # } + # } + }, + 'total_ann_count': , + 'items_missing_annotation': , # [, ] + + ## statistics for classification task + 'items_with_multiple_labels': , # [, ] + + ## statistics for detection task + 'items_with_invalid_value': , + # '': {: [ , ], } + # - properties: 'x', 'y', 'width', 'height', + # 'area(wxh)', 'ratio(w/h)', 'short', 'long' + # - 'short' is min(w,h) and 'long' is max(w,h). + 'items_with_negative_length': , + # '': { : { <'width'|'height'>: , }, } + 'bbox_distribution_in_label': , # : + 'bbox_distribution_in_attribute': , + # : {: { : , }, } + 'bbox_distribution_in_dataset_item': , + # '': + + ## statistics for segmentation task + 'items_with_invalid_value'] = , + # '': {: [ , ], } + # - properties: 'area', 'width', 'height' + 'mask_distribution_in_label'] = , # : + 'mask_distribution_in_attribute'] = , + # : { + # : { : , } + # } + 'mask_distribution_in_dataset_item'] = , + # '': + }, + 'validation_reports': , #[ , ] + # validation_error_format = { + # 'anomaly_type': , # see datumaro/components/errors.py + # 'description': , # see datumaro/components/errors.py + # 'severity': , # 'warning' or 'error' + # 'item_id': , # optional, when it is related to a DatasetItem + # 'subset': , # optional, when it is related to a DatasetItem + # } + 'summary': { + 'errors': , + 'warnings': + } +} + +``` + +`item_key` is defined as, +``` python +item_key = (, ) +``` + +`bbox_template` and `mask_template` are defined as, + +``` python +bbox_template = { + 'width': , + 'height': , + 'area(wxh)': , + 'ratio(w/h)': , + 'short': , # short = min(w, h) + 'long': # long = max(w, h) +} +mask_template = { + 'area': , + 'width': , + 'height': +} +``` + +`numerical_stat_template` is defined as, + +``` python +numerical_stat_template = { + 'items_far_from_mean': , + # {'': {: , }, } + 'mean': , + 'stdev': , + 'min': , + 'max': , + 'median': , + 'histogram': { + 'bins': , # [, ] + 'counts': , # [, ] + } +} +``` + +
+ + ### Register model Supported models: From 962ade03232aefba40c7adef92635cb6557adc59 Mon Sep 17 00:00:00 2001 From: Jihyeon Yi Date: Mon, 10 May 2021 21:04:13 +0900 Subject: [PATCH 18/40] add validation item description (#237) --- docs/user_manual.md | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/user_manual.md b/docs/user_manual.md index cee40321dc..80e31233be 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -904,7 +904,33 @@ datum validate --help datum validate -p ``` -Validation Result: +Here is the list of validation items(a.k.a. anomaly types). + +| Anomaly Type | Description | Task Type | +| ------------ | ----------- | --------- | +| MissingLabelCategories | Metadata (ex. LabelCategories) should be defined | common | +| MissingAnnotation | No annotation found for an Item | common | +| MissingAttribute | An attribute key is missing for an Item | common | +| MultiLabelAnnotations | Item needs a single label | classification | +| UndefinedLabel | A label not defined in the metadata is found for an item | common | +| UndefinedAttribute | An attribute not defined in the metadata is found for an item | common | +| LabelDefinedButNotFound | A label is defined, but not found actually | common | +| AttributeDefinedButNotFound | An attribute is defined, but not found actually | common | +| OnlyOneLabel | The dataset consists of only label | common | +| OnlyOneAttributeValue | The dataset consists of only attribute value | common | +| FewSamplesInLabel | The number of samples in a label might be too low | common | +| FewSamplesInAttribute | The number of samples in an attribute might be too low | common | +| ImbalancedLabels | There is an imbalance in the label distribution | common | +| ImbalancedAttribute | There is an imbalance in the attribute distribution | common | +| ImbalancedDistInLabel | Values (ex. bbox width) are not evenly distributed for a label | detection, segmentation | +| ImbalancedDistInAttribute | Values (ex. bbox width) are not evenly distributed for an attribute | detection, segmentation | +| NegativeLength | The width or height of bounding box is negative | detection | +| InvalidValue | There's invalid (ex. inf, nan) value for bounding box info. | detection | +| FarFromLabelMean | An annotation has an too small or large value than average for a label | detection, segmentation | +| FarFromAttrMean | An annotation has an too small or large value than average for an attribute | detection, segmentation | + + +Validation Result Format:
@@ -957,21 +983,21 @@ Validation Result: # '': ## statistics for segmentation task - 'items_with_invalid_value'] = , + 'items_with_invalid_value': , # '': {: [ , ], } # - properties: 'area', 'width', 'height' - 'mask_distribution_in_label'] = , # : - 'mask_distribution_in_attribute'] = , + 'mask_distribution_in_label': , # : + 'mask_distribution_in_attribute': , # : { # : { : , } # } - 'mask_distribution_in_dataset_item'] = , + 'mask_distribution_in_dataset_item': , # '': }, - 'validation_reports': , #[ , ] + 'validation_reports': , # [ , ] # validation_error_format = { - # 'anomaly_type': , # see datumaro/components/errors.py - # 'description': , # see datumaro/components/errors.py + # 'anomaly_type': , + # 'description': , # 'severity': , # 'warning' or 'error' # 'item_id': , # optional, when it is related to a DatasetItem # 'subset': , # optional, when it is related to a DatasetItem @@ -1027,7 +1053,6 @@ numerical_stat_template = {
- ### Register model Supported models: From f28d622c9e700c338bb03c5d87208f0882ae5185 Mon Sep 17 00:00:00 2001 From: Kirill Sizov Date: Thu, 13 May 2021 11:43:07 +0300 Subject: [PATCH 19/40] Fix converter for Pascal VOC format (#239) --- CHANGELOG.md | 1 + datumaro/plugins/voc_format/converter.py | 2 +- tests/test_voc_format.py | 28 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b775ee01..811043403d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a label "face" for bounding boxes in Wider Face () - Allowed adding "difficult", "truncated", "occluded" attributes when converting to Pascal VOC if these attributes are not present () - Empty lines in YOLO annotations are ignored () +- Export in VOC format when no image info is available () ### Security - diff --git a/datumaro/plugins/voc_format/converter.py b/datumaro/plugins/voc_format/converter.py index 4a5d36b9f6..54be318b0e 100644 --- a/datumaro/plugins/voc_format/converter.py +++ b/datumaro/plugins/voc_format/converter.py @@ -201,7 +201,7 @@ def save_subsets(self): ET.SubElement(source_elem, 'annotation').text = 'Unknown' ET.SubElement(source_elem, 'image').text = 'Unknown' - if item.has_image: + if item.has_image and item.image.has_size: h, w = item.image.size size_elem = ET.SubElement(root_elem, 'size') ET.SubElement(size_elem, 'width').text = str(w) diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index fd03a35eb7..e422f15f4c 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -763,4 +763,30 @@ def test_inplace_save_writes_only_updated_data(self): self.assertFalse(osp.isfile( osp.join(path, 'SegmentationObject', '3.png'))) self.assertFalse(osp.isfile( - osp.join(path, 'SegmentationClass', '3.png'))) \ No newline at end of file + osp.join(path, 'SegmentationClass', '3.png'))) + + def test_can_save_dataset_with_no_data_images(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='frame1', subset='test', + image=Image(path='frame1.jpg'), + annotations=[ + Bbox(1.0, 2.0, 3.0, 4.0, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False + }, + id=1, label=0, group=1 + ) + ] + ) + ]) + + def categories(self): + return VOC.make_voc_categories() + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc'), test_dir) From ef003cae34b5d8dc89d39290fa01132187172472 Mon Sep 17 00:00:00 2001 From: Kirill Sizov Date: Fri, 14 May 2021 13:57:03 +0300 Subject: [PATCH 20/40] User documentation for Pascal VOC format (#228) * add user documentation for Pascal VOC format * add integration tests * update changelog --- CHANGELOG.md | 1 + docs/pascal_voc_user_manual.md | 317 ++++++++++++++++++ docs/user_manual.md | 1 + .../Annotations/2007_000001.xml | 0 .../ImageSets/Action/test.txt | 0 .../ImageSets/Action/train.txt | 0 .../ImageSets/Layout/test.txt | 0 .../ImageSets/Layout/train.txt | 0 .../ImageSets/Main/aeroplane_train.txt | 0 .../ImageSets/Main/background_train.txt | 0 .../ImageSets/Main/bicycle_train.txt | 0 .../ImageSets/Main/bird_train.txt | 0 .../ImageSets/Main/boat_train.txt | 0 .../ImageSets/Main/bottle_train.txt | 0 .../ImageSets/Main/bus_train.txt | 0 .../ImageSets/Main/car_train.txt | 0 .../ImageSets/Main/cat_train.txt | 0 .../ImageSets/Main/chair_train.txt | 0 .../ImageSets/Main/cow_train.txt | 0 .../ImageSets/Main/diningtable_train.txt | 0 .../ImageSets/Main/dog_train.txt | 0 .../ImageSets/Main/horse_train.txt | 0 .../ImageSets/Main/ignored_train.txt | 0 .../ImageSets/Main/motorbike_train.txt | 0 .../ImageSets/Main/person_train.txt | 0 .../ImageSets/Main/pottedplant_train.txt | 0 .../ImageSets/Main/sheep_train.txt | 0 .../ImageSets/Main/sofa_train.txt | 0 .../ImageSets/Main/test.txt | 0 .../ImageSets/Main/train.txt | 0 .../ImageSets/Main/train_train.txt | 0 .../ImageSets/Main/tvmonitor_train.txt | 0 .../ImageSets/Segmentation/test.txt | 0 .../ImageSets/Segmentation/train.txt | 0 .../JPEGImages/2007_000002.jpg | Bin .../SegmentationClass/2007_000001.png | Bin .../SegmentationObject/2007_000001.png | Bin .../voc_dataset2/Annotations/a.xml | 22 ++ .../voc_dataset2/Annotations/b.xml | 22 ++ .../voc_dataset2/Annotations/c.xml | 22 ++ .../voc_dataset2/Annotations/d.xml | 22 ++ .../ImageSets/Action/trainval.txt | 4 + .../ImageSets/Layout/trainval.txt | 4 + .../ImageSets/Main/aeroplane_trainval.txt | 0 .../ImageSets/Main/background_trainval.txt | 0 .../ImageSets/Main/bicycle_trainval.txt | 0 .../ImageSets/Main/bird_trainval.txt | 0 .../ImageSets/Main/boat_trainval.txt | 0 .../ImageSets/Main/bottle_trainval.txt | 0 .../ImageSets/Main/bus_trainval.txt | 0 .../ImageSets/Main/car_trainval.txt | 0 .../ImageSets/Main/cat_trainval.txt | 0 .../ImageSets/Main/chair_trainval.txt | 0 .../ImageSets/Main/cow_trainval.txt | 0 .../ImageSets/Main/diningtable_trainval.txt | 0 .../ImageSets/Main/dog_trainval.txt | 0 .../ImageSets/Main/horse_trainval.txt | 0 .../ImageSets/Main/ignored_trainval.txt | 0 .../ImageSets/Main/motorbike_trainval.txt | 0 .../ImageSets/Main/person_trainval.txt | 0 .../ImageSets/Main/pottedplant_trainval.txt | 0 .../ImageSets/Main/sheep_trainval.txt | 0 .../ImageSets/Main/sofa_trainval.txt | 0 .../ImageSets/Main/train_trainval.txt | 0 .../voc_dataset2/ImageSets/Main/trainval.txt | 4 + .../ImageSets/Main/tvmonitor_trainval.txt | 0 .../ImageSets/Segmentation/trainval.txt | 0 .../voc_dataset/voc_dataset2/labelmap.txt | 23 ++ tests/cli/test_voc_format.py | 282 ++++++++++++++++ tests/test_voc_format.py | 2 +- 70 files changed, 725 insertions(+), 1 deletion(-) create mode 100644 docs/pascal_voc_user_manual.md rename tests/assets/voc_dataset/{ => voc_dataset1}/Annotations/2007_000001.xml (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Action/test.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Action/train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Layout/test.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Layout/train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/aeroplane_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/background_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/bicycle_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/bird_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/boat_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/bottle_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/bus_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/car_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/cat_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/chair_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/cow_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/diningtable_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/dog_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/horse_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/ignored_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/motorbike_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/person_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/pottedplant_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/sheep_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/sofa_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/test.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/train_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Main/tvmonitor_train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Segmentation/test.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/ImageSets/Segmentation/train.txt (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/JPEGImages/2007_000002.jpg (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/SegmentationClass/2007_000001.png (100%) rename tests/assets/voc_dataset/{ => voc_dataset1}/SegmentationObject/2007_000001.png (100%) create mode 100644 tests/assets/voc_dataset/voc_dataset2/Annotations/a.xml create mode 100644 tests/assets/voc_dataset/voc_dataset2/Annotations/b.xml create mode 100644 tests/assets/voc_dataset/voc_dataset2/Annotations/c.xml create mode 100644 tests/assets/voc_dataset/voc_dataset2/Annotations/d.xml create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/aeroplane_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/background_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bicycle_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bird_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/boat_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bottle_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bus_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/car_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cat_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/chair_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cow_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/diningtable_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/dog_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/horse_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/ignored_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/motorbike_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/person_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/pottedplant_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sheep_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sofa_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/train_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/tvmonitor_trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/ImageSets/Segmentation/trainval.txt create mode 100644 tests/assets/voc_dataset/voc_dataset2/labelmap.txt create mode 100644 tests/cli/test_voc_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 811043403d..dd78dee5bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for Segmentation Splitting () - Support for CIFAR-10/100 dataset format () - Support COCO panoptic and stuff format () +- Documentation file and integration tests for Pascal VOC format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/docs/pascal_voc_user_manual.md b/docs/pascal_voc_user_manual.md new file mode 100644 index 0000000000..0e8244f44b --- /dev/null +++ b/docs/pascal_voc_user_manual.md @@ -0,0 +1,317 @@ +# Pascal VOC user manual + +## Contents +- [Format specification](#format-specification) +- [Load Pascal VOC dataset](#load-pascal-voc-dataset) +- [Export to other formats](#export-to-other-formats) +- [Export to Pascal VOC](#export-to-pascal-VOC) +- [Particular use cases](#particular-use-cases) + +## Format specification + +- Pascal VOC format specification available +[here](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/devkit_doc.pdf). + +- Original Pascal VOC dataset format support the followoing types of annotations: + - `Labels` (for classification tasks); + - `Bounding boxes` (for detection, action detection and person layout tasks); + - `Masks` (for segmentations tasks). + +- Supported attributes: + - `occluded`: indicates that a significant portion of the object within the + bounding box is occluded by another object; + - `truncated`: indicates that the bounding box specified for the object does + not correspond to the full extent of the object; + - `difficult`: indicates that the object is considered difficult to recognize; + - action attributes (`jumping`, `reading`, `phoning` and + [more](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/actionexamples/index.html)). + +## Load Pascal VOC dataset + +The Pascal VOC dataset is available for free download +[here](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html#devkit) + +There are two ways to create Datumaro project and add Pascal VOC dataset to it: + +``` bash +datum import --format voc --input-path +# or +datum create +datum add path -f voc +``` + +It is possible to specify project name and project directory run +`datum create --help` for more information. +Pascal VOC dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── label_map.txt # list of non-pascal labels (optional) + ├── Annotations/ + │ ├── ann1.xml # Pascal VOC format annotation file + │ ├── ann2.xml + │ ├── ... + ├── JPEGImages/ + │ ├── img1.jpg + │ ├── img2.jpg + │ ├── ... + ├── SegmentationClass/ # directory with semantic segmentation masks + │ ├── img1.png + │ ├── img2.png + │ ├── ... + ├── SegmentationObject/ # directory with instance segmentation masks + │ ├── img1.png + │ ├── img2.png + │ ├── ... + ├── ImageSets/ + │ ├── Main/ # directory with list of images for detection and classification task + │ │ ├── test.txt # list of image names in test subset (without extension) + | | ├── train.txt # list of image names in train subset (without extension) + | | ├── ... + │ ├── Layout/ # directory with list of images for person layout task + │ │ ├── test.txt + | | ├── train.txt + | | ├── ... + │ ├── Action/ # directory with list of images for action classification task + │ │ ├── test.txt + | | ├── train.txt + | | ├── ... + │ ├── Segmentation/ # directory with list of images for segmentation task + │ │ ├── test.txt + | | ├── train.txt + | | ├── ... +``` + +The `ImageSets` directory should contain at least one of the directories: +`Main`, `Layout`, `Action`, `Segmentation`. +These directories contain `.txt` files +with a list of images in a subset, the subset name is the same as the `.txt` file name. + +In `label_map.txt` you can define custom color map and non-pascal labels, for example: + +``` +# label_map [label : color_rgb : parts : actions] +helicopter::: +elephant:0:124:134:head,ear,foot: +``` +It is also possible to import grayscale (1-channel) PNG masks. +For grayscale masks provide a list of labels with the number of lines +equal to the maximum color index on images. The lines must be in the +right order so that line index is equal to the color index. Lines can +have arbitrary, but different, colors. If there are gaps in the used +color indices in the annotations, they must be filled with arbitrary +dummy labels. Example: + +``` +car:0,128,0:: # color index 0 +aeroplane:10,10,128:: # color index 1 +_dummy2:2,2,2:: # filler for color index 2 +_dummy3:3,3,3:: # filler for color index 3 +boat:108,0,100:: # color index 3 +... +_dummy198:198,198,198:: # filler for color index 198 +_dummy199:199,199,199:: # filler for color index 199 +the_last_label:12,28,0:: # color index 200 +``` + +You can import dataset for specific tasks +of Pascal VOC dataset instead of the whole dataset, +for example: + +``` bash +datum add path -f voc_detection +``` + +Datumaro supports the following Pascal VOC tasks: +- Image classification (`voc_classification`) +- Object detection (`voc_detection`) +- Action classification (`voc_action`) +- Class and instance segmentation (`voc_segmentation`) +- Person layout detection (`voc_layout`) + +To make sure that the selected dataset has been added to the project, you can run +`datum info`, which will display the project and dataset information. + +## Export to other formats + +Datumaro can convert Pascal VOC dataset into any other format +[Datumaro supports](../docs/user_manual.md#supported-formats). + +Such conversion will only be successful if the output +format can represent the type of dataset you want to convert, +e.g. image classification annotations can be +saved in `ImageNet` format, but no as `COCO keypoints`. + +There are few ways to convert Pascal VOC dataset to other dataset format: + +``` bash +datum import -f voc -i +datum export -f coco -o +# or +datum convert -if voc -i -f coco -o + +``` + +Some formats provide extra options for conversion. +These options are passed after double dash (`--`) in the command line. +To get information about them, run + +`datum export -f -- -h` + +## Export to Pascal VOC + +There are few ways to convert an existing dataset to Pascal VOC format: + +``` bash +# export dataset into Pascal VOC format (classification) from existing project +datum export -p -f voc -o -- --tasks classification + +# converting to Pascal VOC format from other format +datum convert -if imagenet -i \ + -f voc -o \ + -- --label_map voc --save-images +``` + +Extra options for export to Pascal VOC format: + +- `--save-images` allow to export dataset with saving images +(by default `False`); + +- `--image-ext IMAGE_EXT` allow to specify image extension +for exporting dataset (by default use original or `.jpg` if none); + +- `--apply-colormap APPLY_COLORMAP` allow to use colormap for class +and instance masks (by default `True`); + +- `--allow-attributes ALLOW_ATTRIBUTES` allow export of attributes +(by default `True`); + +- `--tasks TASKS` allow to specify tasks for export dataset, +by default Datumaro uses all tasks. Example: + +```bash +datum import -o project -f voc -i ./VOC2012 +datum export -p project -f voc -- --tasks detection,classification +``` + +- `--label_map` allow to define a custom colormap. Example + +``` bash +# mycolormap.txt [label : color_rgb : parts : actions]: +# cat:0,0,255:: +# person:255,0,0:head: +datum export -f voc_segmentation -- --label-map mycolormap.txt + +# or you can use original voc colomap: +datum export -f voc_segmentation -- --label-map voc +``` + +## Particular use cases + +Datumaro supports filtering, transformation, merging etc. for all formats +and for the Pascal VOC format in particular. Follow +[user manual](../docs/user_manual.md) +to get more information about these operations. + +There are few examples of using Datumaro operations to solve +particular problems with Pascal VOC dataset: + +### Example 1. How to prepare an original dataset for training. +In this example, preparing the original dataset to train the semantic segmentation model includes: +loading, +checking duplicate images, +setting the number of images, +splitting into subsets, +export the result to Pascal VOC format. + +```bash +datum create -o project +datum add path -p project -f voc_segmentation ./VOC2012/ImageSets/Segmentation/trainval.txt +datum stats -p project # check statisctics.json -> repeated images +datum transform -p project -o ndr_project -t ndr -- -w trainval -k 2500 +datum filter -p ndr_project -o trainval2500 -e '/item[subset="trainval"]' +datum transform -p trainval2500 -o final_project -t random_split -- -s train:.8 -s val:.2 +datum export -p final_project -o dataset -f voc -- --label-map voc --save-images +``` + +### Example 2. How to create custom dataset + +```python +from datumaro.components.dataset import Dataset +from datumaro.util.image import Image +from datumaro.components.extractor import Bbox, Polygon, Label, DatasetItem + +dataset = Dataset.from_iterable([ + DatasetItem(id='image1', image=Image(path='image1.jpg', size=(10, 20)), + annotations=[Label(3), + Bbox(1.0, 1.0, 10.0, 8.0, label=0, attributes={'difficult': True, 'running': True}), + Polygon([1, 2, 3, 2, 4, 4], label=2, attributes={'occluded': True}), + Polygon([6, 7, 8, 8, 9, 7, 9, 6], label=2), + ] + ), +], categories=['person', 'sky', 'water', 'lion']) + +dataset.transform('polygons_to_masks') +dataset.export('./mydataset', format='voc', label_map='my_labelmap.txt') + +""" +my_labelmap.txt: +# label:color_rgb:parts:actions +person:0,0,255:hand,foot:jumping,running +sky:128,0,0:: +water:0,128,0:: +lion:255,128,0:: +""" +``` + +### Example 3. Load, filter and convert from code +Load Pascal VOC dataset, and export train subset with items +which has `jumping` attribute: + +```python +from datumaro.components.dataset import Dataset + +dataset = Dataset.import_from('./VOC2012', format='voc') + +train_dataset = dataset.get_subset('train').as_dataset() + +def only_jumping(item): + for ann in item.annotations: + if ann.attributes.get('jumping'): + return True + return False + +train_dataset.select(only_jumping) + +train_dataset.export('./jumping_label_me', format='label_me', save_images=True) +``` + +### Example 4. Get information about items in Pascal VOC 2012 dataset for segmentation task: + +```python +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import AnnotationType + +dataset = Dataset.import_from('./VOC2012', format='voc') + +def has_mask(item): + for ann in item.annotations: + if ann.type == AnnotationType.mask: + return True + return False + +dataset.select(has_mask) + +print("Pascal VOC 2012 has %s images for segmentation task:" % len(dataset)) +for subset_name, subset in dataset.subsets().items(): + for item in subset: + print(item.id, subset_name, end=";") +``` + +After executing this code, we can see that there are 5826 images +in Pascal VOC 2012 has for segmentation task and this result is the same as the +[official documentation](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/dbstats.html) + +Some examples of working with Pascal VOC dataset from code you can found in +[tests](../tests/test_voc_format.py) \ No newline at end of file diff --git a/docs/user_manual.md b/docs/user_manual.md index 80e31233be..0f1f57b336 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -92,6 +92,7 @@ List of supported formats: - PASCAL VOC (`classification`, `detection`, `segmentation` (class, instances), `action_classification`, `person_layout`) - [Format specification](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/htmldoc/index.html) - [Dataset example](../tests/assets/voc_dataset) + - [Format documentation](./pascal_voc_user_manual.md) - YOLO (`bboxes`) - [Format specification](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) - [Dataset example](../tests/assets/yolo_dataset) diff --git a/tests/assets/voc_dataset/Annotations/2007_000001.xml b/tests/assets/voc_dataset/voc_dataset1/Annotations/2007_000001.xml similarity index 100% rename from tests/assets/voc_dataset/Annotations/2007_000001.xml rename to tests/assets/voc_dataset/voc_dataset1/Annotations/2007_000001.xml diff --git a/tests/assets/voc_dataset/ImageSets/Action/test.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Action/test.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Action/test.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Action/test.txt diff --git a/tests/assets/voc_dataset/ImageSets/Action/train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Action/train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Action/train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Action/train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Layout/test.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Layout/test.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Layout/test.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Layout/test.txt diff --git a/tests/assets/voc_dataset/ImageSets/Layout/train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Layout/train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Layout/train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Layout/train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/aeroplane_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/aeroplane_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/background_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/background_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/background_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/background_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bicycle_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bicycle_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/bird_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bird_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/bird_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bird_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/boat_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/boat_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/boat_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/boat_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bottle_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bottle_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/bus_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bus_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/bus_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/bus_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/car_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/car_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/car_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/car_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/cat_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/cat_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/cat_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/cat_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/chair_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/chair_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/chair_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/chair_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/cow_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/cow_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/cow_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/cow_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/diningtable_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/diningtable_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/dog_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/dog_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/dog_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/dog_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/horse_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/horse_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/horse_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/horse_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/ignored_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/ignored_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/motorbike_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/motorbike_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/person_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/person_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/person_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/person_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/pottedplant_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/pottedplant_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/sheep_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/sheep_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/sofa_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/sofa_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/test.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/test.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/test.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/test.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/train_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/train_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/train_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/train_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/tvmonitor_train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Main/tvmonitor_train.txt diff --git a/tests/assets/voc_dataset/ImageSets/Segmentation/test.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Segmentation/test.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Segmentation/test.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Segmentation/test.txt diff --git a/tests/assets/voc_dataset/ImageSets/Segmentation/train.txt b/tests/assets/voc_dataset/voc_dataset1/ImageSets/Segmentation/train.txt similarity index 100% rename from tests/assets/voc_dataset/ImageSets/Segmentation/train.txt rename to tests/assets/voc_dataset/voc_dataset1/ImageSets/Segmentation/train.txt diff --git a/tests/assets/voc_dataset/JPEGImages/2007_000002.jpg b/tests/assets/voc_dataset/voc_dataset1/JPEGImages/2007_000002.jpg similarity index 100% rename from tests/assets/voc_dataset/JPEGImages/2007_000002.jpg rename to tests/assets/voc_dataset/voc_dataset1/JPEGImages/2007_000002.jpg diff --git a/tests/assets/voc_dataset/SegmentationClass/2007_000001.png b/tests/assets/voc_dataset/voc_dataset1/SegmentationClass/2007_000001.png similarity index 100% rename from tests/assets/voc_dataset/SegmentationClass/2007_000001.png rename to tests/assets/voc_dataset/voc_dataset1/SegmentationClass/2007_000001.png diff --git a/tests/assets/voc_dataset/SegmentationObject/2007_000001.png b/tests/assets/voc_dataset/voc_dataset1/SegmentationObject/2007_000001.png similarity index 100% rename from tests/assets/voc_dataset/SegmentationObject/2007_000001.png rename to tests/assets/voc_dataset/voc_dataset1/SegmentationObject/2007_000001.png diff --git a/tests/assets/voc_dataset/voc_dataset2/Annotations/a.xml b/tests/assets/voc_dataset/voc_dataset2/Annotations/a.xml new file mode 100644 index 0000000000..d0c631a222 --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/Annotations/a.xml @@ -0,0 +1,22 @@ + + + a.jpg + + Unknown + Unknown + Unknown + + 0 + + background + 0 + 1 + 0 + + 1.0 + 2.0 + 4.0 + 6.0 + + + diff --git a/tests/assets/voc_dataset/voc_dataset2/Annotations/b.xml b/tests/assets/voc_dataset/voc_dataset2/Annotations/b.xml new file mode 100644 index 0000000000..4dbfb5646f --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/Annotations/b.xml @@ -0,0 +1,22 @@ + + + b.jpg + + Unknown + Unknown + Unknown + + 0 + + aeroplane + 0 + 1 + 0 + + 2.0 + 2.0 + 7.0 + 6.0 + + + diff --git a/tests/assets/voc_dataset/voc_dataset2/Annotations/c.xml b/tests/assets/voc_dataset/voc_dataset2/Annotations/c.xml new file mode 100644 index 0000000000..72071892ca --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/Annotations/c.xml @@ -0,0 +1,22 @@ + + + c.jpg + + Unknown + Unknown + Unknown + + 0 + + bicycle + 0 + 0 + 0 + + 3.0 + 1.0 + 11.0 + 6.0 + + + diff --git a/tests/assets/voc_dataset/voc_dataset2/Annotations/d.xml b/tests/assets/voc_dataset/voc_dataset2/Annotations/d.xml new file mode 100644 index 0000000000..8917c1b7d5 --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/Annotations/d.xml @@ -0,0 +1,22 @@ + + + d.jpg + + Unknown + Unknown + Unknown + + 0 + + bird + 0 + 0 + 0 + + 4.0 + 4.0 + 8.0 + 8.0 + + + diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt new file mode 100644 index 0000000000..d68dd4031d --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt @@ -0,0 +1,4 @@ +a +b +c +d diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt new file mode 100644 index 0000000000..d68dd4031d --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt @@ -0,0 +1,4 @@ +a +b +c +d diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/aeroplane_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/aeroplane_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/background_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/background_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bicycle_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bicycle_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bird_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bird_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/boat_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/boat_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bottle_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bottle_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bus_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/bus_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/car_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/car_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cat_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cat_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/chair_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/chair_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cow_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/cow_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/diningtable_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/diningtable_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/dog_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/dog_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/horse_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/horse_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/ignored_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/ignored_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/motorbike_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/motorbike_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/person_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/person_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/pottedplant_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/pottedplant_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sheep_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sheep_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sofa_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/sofa_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/train_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/train_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt new file mode 100644 index 0000000000..d68dd4031d --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt @@ -0,0 +1,4 @@ +a +b +c +d diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/tvmonitor_trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/tvmonitor_trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Segmentation/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Segmentation/trainval.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/assets/voc_dataset/voc_dataset2/labelmap.txt b/tests/assets/voc_dataset/voc_dataset2/labelmap.txt new file mode 100644 index 0000000000..28c44bd6a5 --- /dev/null +++ b/tests/assets/voc_dataset/voc_dataset2/labelmap.txt @@ -0,0 +1,23 @@ +# label:color_rgb:parts:actions +background:0,0,0:: +aeroplane:128,0,0:: +bicycle:0,128,0:: +bird:128,128,0:: +boat:0,0,128:: +bottle:128,0,128:: +bus:0,128,128:: +car:128,128,128:: +cat:64,0,0:: +chair:192,0,0:: +cow:64,128,0:: +diningtable:192,128,0:: +dog:64,0,128:: +horse:192,0,128:: +motorbike:64,128,128:: +person:192,128,128:: +pottedplant:0,64,0:: +sheep:128,64,0:: +sofa:0,192,0:: +train:128,192,0:: +tvmonitor:0,64,128:: +ignored:224,224,192:: diff --git a/tests/cli/test_voc_format.py b/tests/cli/test_voc_format.py new file mode 100644 index 0000000000..c1516bcda2 --- /dev/null +++ b/tests/cli/test_voc_format.py @@ -0,0 +1,282 @@ +import os.path as osp +import numpy as np +from collections import OrderedDict + +from unittest import TestCase + +import datumaro.plugins.voc_format.format as VOC +from datumaro.components.dataset import Dataset, DatasetItem +from datumaro.components.extractor import Bbox, Mask, Image, Label +from datumaro.cli.__main__ import main +from datumaro.util.test_utils import TestDir, compare_datasets + +DUMMY_DATASETS_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'voc_dataset') + +def run(test, *args, expected_code=0): + test.assertEqual(expected_code, main(args), str(args)) + +class VocIntegrationScenarios(TestCase): + def _test_can_save_and_load(self, project_path, source_path, source_dataset, + dataset_format, result_path=None, label_map=None): + run(self, 'create', '-o', project_path) + run(self, 'add', 'path', '-p', project_path, '-f', dataset_format, source_path) + + result_dir = osp.join(project_path, 'voc_dataset') + run(self, 'export', '-f', dataset_format, '-p', project_path, + '-o', result_dir, '--', '--label-map', label_map) + + result_path = osp.join(result_dir, result_path) if result_path else result_dir + target_dataset = Dataset.import_from(result_path, dataset_format) + compare_datasets(self, source_dataset, target_dataset) + + def test_preparing_dataset_for_train_model(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='c', subset='train', + annotations=[ + Bbox(3.0, 1.0, 8.0, 5.0, + attributes={ + 'truncated': False, + 'occluded': False, + 'difficult': False + }, + id=1, label=2, group=1 + ) + ] + ), + DatasetItem(id='d', subset='test', + annotations=[ + Bbox(4.0, 4.0, 4.0, 4.0, + attributes={ + 'truncated': False, + 'occluded': False, + 'difficult': False + }, + id=1, label=3, group=1 + ) + ] + ) + ], categories=VOC.make_voc_categories()) + + dataset_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset2') + + with TestDir() as test_dir: + run(self, 'create', '-o', test_dir) + run(self, 'add', 'path', '-p', test_dir, '-f', 'voc', dataset_path) + + result_path = osp.join(test_dir, 'result') + run(self, 'filter', '-p', test_dir, '-m', 'i+a', + '-e', "/item/annotation[occluded='False']", '-o', result_path) + + splitted_path = osp.join(test_dir, 'splitted') + run(self, 'transform', '-p', result_path, '-o', splitted_path, + '-t', 'random_split', '--', '-s', 'test:.5', + '-s', 'train:.5', '--seed', '1') + + export_path = osp.join(test_dir, 'dataset') + run(self, 'export', '-p', splitted_path, '-f', 'voc', + '-o', export_path, '--', '--label-map', 'voc') + + parsed_dataset = Dataset.import_from(export_path, format='voc') + compare_datasets(self, source_dataset, parsed_dataset) + + def test_convert_to_voc_format(self): + label_map = OrderedDict(('label_' + str(i), [None, [], []]) for i in range(10)) + label_map['background'] = [None, [], []] + label_map.move_to_end('background', last=False) + + source_dataset = Dataset.from_iterable([ + DatasetItem(id='1', subset='train', + annotations=[ + Bbox(0.0, 2.0, 4.0, 2.0, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False + }, + id=1, label=3, group=1 + ), + Bbox(3.0, 3.0, 2.0, 3.0, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False + }, + id=2, label=5, group=2 + ) + ] + ) + ], categories=VOC.make_voc_categories(label_map)) + + with TestDir() as test_dir: + yolo_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'yolo_dataset') + + run(self, 'create', '-o', test_dir) + run(self, 'add', 'path', '-p', test_dir, '-f', 'yolo', yolo_dir) + + voc_export = osp.join(test_dir, 'voc_export') + run(self, 'export', '-p', test_dir, '-f', 'voc', + '-o', voc_export) + + parsed_dataset = Dataset.import_from(voc_export, format='voc') + compare_datasets(self, source_dataset, parsed_dataset) + + def test_can_save_and_load_voc_dataset(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + image=Image(path='2007_000001.jpg', size=(10, 20)), + annotations=[Label(i) for i in range(22) if i % 2 == 1] + [ + Bbox(4.0, 5.0, 2.0, 2.0, label=15, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False, + **{ + a.name : a.value % 2 == 1 + for a in VOC.VocAction + } + }, + id=1, group=1 + ), + Bbox(1.0, 2.0, 2.0, 2.0, label=8, + attributes={ + 'difficult': False, + 'truncated': True, + 'occluded': False, + 'pose': 'Unspecified' + }, + id=2, group=2 + ), + Bbox(5.5, 6.0, 2.0, 2.0, label=22, + id=0, group=1 + ), + Mask(image=np.ones([5, 10]), label=2, group=1) + ] + ), + DatasetItem(id='2007_000002', subset='test', + image=np.ones((10, 20, 3)) + ) + ], categories=VOC.make_voc_categories()) + + voc_dir = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1') + with TestDir() as test_dir: + self._test_can_save_and_load(test_dir, voc_dir, source_dataset, + 'voc', label_map='voc') + + def test_can_save_and_load_voc_layout_dataset(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + image=Image(path='2007_000001.jpg', size=(10, 20)), + annotations=[ + Bbox(4.0, 5.0, 2.0, 2.0, label=15, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False, + **{ + a.name : a.value % 2 == 1 + for a in VOC.VocAction + } + }, + id=1, group=1 + ), + Bbox(5.5, 6.0, 2.0, 2.0, label=22, + id=0, group=1 + ), + ] + ), + ], categories=VOC.make_voc_categories()) + + voc_layout_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1', + 'ImageSets', 'Layout', 'train.txt') + + with TestDir() as test_dir: + result_voc_path = osp.join('ImageSets', 'Layout', 'train.txt') + self._test_can_save_and_load(test_dir, voc_layout_path, source_dataset, + 'voc_layout', result_path=result_voc_path, label_map='voc') + + def test_can_save_and_load_voc_detect_dataset(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + image=Image(path='2007_000001.jpg', size=(10, 20)), + annotations=[ + Bbox(4.0, 5.0, 2.0, 2.0, label=15, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False, + **{ + a.name : a.value % 2 == 1 + for a in VOC.VocAction + } + }, + id=2, group=2 + ), + Bbox(1.0, 2.0, 2.0, 2.0, label=8, + attributes={ + 'difficult': False, + 'truncated': True, + 'occluded': False, + 'pose': 'Unspecified' + }, + id=1, group=1 + ) + ] + ), + ], categories=VOC.make_voc_categories()) + + voc_detection_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1', + 'ImageSets', 'Main', 'train.txt') + + with TestDir() as test_dir: + result_voc_path = osp.join('ImageSets', 'Main', 'train.txt') + self._test_can_save_and_load(test_dir, voc_detection_path, source_dataset, + 'voc_detection', result_path=result_voc_path, label_map='voc') + + def test_can_save_and_load_voc_segmentation_dataset(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + image=Image(path='2007_000001.jpg', size=(10, 20)), + annotations=[ + Mask(image=np.ones([5, 10]), label=2, group=1) + ] + ) + ], categories=VOC.make_voc_categories()) + + voc_segm_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1', + 'ImageSets', 'Segmentation', 'train.txt') + + with TestDir() as test_dir: + result_voc_path = osp.join('ImageSets', 'Segmentation', 'train.txt') + self._test_can_save_and_load(test_dir, voc_segm_path, source_dataset, + 'voc_segmentation', result_path=result_voc_path, label_map='voc') + + def test_can_save_and_load_voc_action_dataset(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + image=Image(path='2007_000001.jpg', size=(10, 20)), + annotations=[ + Bbox(4.0, 5.0, 2.0, 2.0, label=15, + attributes={ + 'difficult': False, + 'truncated': False, + 'occluded': False, + **{ + a.name : a.value % 2 == 1 + for a in VOC.VocAction + } + }, + id=1, group=1 + ) + ] + ) + ], categories=VOC.make_voc_categories()) + + voc_act_path = osp.join(DUMMY_DATASETS_DIR, 'voc_dataset1', + 'ImageSets', 'Action', 'train.txt') + + with TestDir() as test_dir: + result_voc_path = osp.join('ImageSets', 'Action', 'train.txt') + self._test_can_save_and_load(test_dir, voc_act_path, source_dataset, + 'voc_action', result_path=result_voc_path, label_map='voc') diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index e422f15f4c..5f30ccd1f9 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -76,7 +76,7 @@ def categories(self): return VOC.make_voc_categories() -DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'voc_dataset') +DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'voc_dataset', 'voc_dataset1') class VocImportTest(TestCase): def test_can_import(self): From ca9f78ee8291fada64bbfc0be95a87b1f2fb81cc Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Fri, 14 May 2021 14:16:12 +0300 Subject: [PATCH 21/40] Support for MNIST dataset format (#234) * add mnist format * add mnist csv format * add mnist to documentation --- CHANGELOG.md | 1 + README.md | 2 + datumaro/plugins/mnist_csv_format.py | 170 ++++++++++++++ datumaro/plugins/mnist_format.py | 209 ++++++++++++++++++ docs/mnist_user_manual.md | 176 +++++++++++++++ docs/user_manual.md | 8 + tests/assets/mnist_csv_dataset/mnist_test.csv | 3 + .../assets/mnist_csv_dataset/mnist_train.csv | 2 + .../mnist_dataset/t10k-images-idx3-ubyte.gz | Bin 0 -> 74 bytes .../mnist_dataset/t10k-labels-idx1-ubyte.gz | Bin 0 -> 54 bytes .../mnist_dataset/train-images-idx3-ubyte.gz | Bin 0 -> 70 bytes .../mnist_dataset/train-labels-idx1-ubyte.gz | Bin 0 -> 54 bytes tests/test_mnist_csv_format.py | 185 ++++++++++++++++ tests/test_mnist_format.py | 184 +++++++++++++++ 14 files changed, 940 insertions(+) create mode 100644 datumaro/plugins/mnist_csv_format.py create mode 100644 datumaro/plugins/mnist_format.py create mode 100644 docs/mnist_user_manual.md create mode 100644 tests/assets/mnist_csv_dataset/mnist_test.csv create mode 100644 tests/assets/mnist_csv_dataset/mnist_train.csv create mode 100644 tests/assets/mnist_dataset/t10k-images-idx3-ubyte.gz create mode 100644 tests/assets/mnist_dataset/t10k-labels-idx1-ubyte.gz create mode 100644 tests/assets/mnist_dataset/train-images-idx3-ubyte.gz create mode 100644 tests/assets/mnist_dataset/train-labels-idx1-ubyte.gz create mode 100644 tests/test_mnist_csv_format.py create mode 100644 tests/test_mnist_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dd78dee5bc..d177cf58a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for CIFAR-10/100 dataset format () - Support COCO panoptic and stuff format () - Documentation file and integration tests for Pascal VOC format () +- Support for MNIST and MNIST in CSV dataset formats () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/README.md b/README.md index 00697b4bbe..7bd9d6252b 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,8 @@ CVAT annotations ---> Publication, statistics etc. - [MOTS PNG](https://www.vision.rwth-aachen.de/page/mots) - [ImageNet](http://image-net.org/) - [CIFAR-10/100](https://www.cs.toronto.edu/~kriz/cifar.html) (`classification`) + - [MNIST](http://yann.lecun.com/exdb/mnist/) (`classification`) + - [MNIST in CSV](https://pjreddie.com/projects/mnist-in-csv/) (`classification`) - [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [CVAT](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) - [LabelMe](http://labelme.csail.mit.edu/Release3.0) diff --git a/datumaro/plugins/mnist_csv_format.py b/datumaro/plugins/mnist_csv_format.py new file mode 100644 index 0000000000..ae0fa8bf8c --- /dev/null +++ b/datumaro/plugins/mnist_csv_format.py @@ -0,0 +1,170 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +import os.path as osp + +import numpy as np +from datumaro.components.converter import Converter +from datumaro.components.extractor import (AnnotationType, DatasetItem, + Importer, Label, LabelCategories, SourceExtractor) + + +class MnistCsvPath: + IMAGE_SIZE = 28 + NONE_LABEL = -1 + +class MnistCsvExtractor(SourceExtractor): + def __init__(self, path, subset=None): + if not osp.isfile(path): + raise FileNotFoundError("Can't read annotation file '%s'" % path) + + if not subset: + file_name = osp.splitext(osp.basename(path))[0] + subset = file_name.rsplit('_', maxsplit=1)[-1] + + super().__init__(subset=subset) + self._dataset_dir = osp.dirname(path) + + self._categories = self._load_categories() + + self._items = list(self._load_items(path).values()) + + def _load_categories(self): + label_cat = LabelCategories() + + labels_file = osp.join(self._dataset_dir, 'labels.txt') + if osp.isfile(labels_file): + with open(labels_file, encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + label_cat.add(line) + else: + for i in range(10): + label_cat.add(str(i)) + + return { AnnotationType.label: label_cat } + + def _load_items(self, path): + items = {} + with open(path, 'r', encoding='utf-8') as f: + annotation_table = f.readlines() + + metafile = osp.join(self._dataset_dir, 'meta_%s.csv' % self._subset) + meta = [] + if osp.isfile(metafile): + with open(metafile, 'r', encoding='utf-8') as f: + meta = f.readlines() + + for i, data in enumerate(annotation_table): + data = data.split(',') + item_anno = [] + label = int(data[0]) + if label != MnistCsvPath.NONE_LABEL: + item_anno.append(Label(label)) + + if 0 < len(meta): + meta[i] = meta[i].strip().split(',') + + # support for single-channel image only + image = None + if 1 < len(data): + if 0 < len(meta) and 1 < len(meta[i]): + image = np.array([int(pix) for pix in data[1:]], + dtype='uint8').reshape(int(meta[i][-2]), int(meta[i][-1])) + else: + image = np.array([int(pix) for pix in data[1:]], + dtype='uint8').reshape(28, 28) + + if 0 < len(meta) and len(meta[i]) in [1, 3]: + i = meta[i][0] + + items[i] = DatasetItem(id=i, subset=self._subset, + image=image, annotations=item_anno) + return items + +class MnistCsvImporter(Importer): + @classmethod + def find_sources(cls, path): + return cls._find_sources_recursive(path, '.csv', 'mnist_csv', + file_filter=lambda p: not osp.basename(p).startswith('meta')) + +class MnistCsvConverter(Converter): + DEFAULT_IMAGE_EXT = '.png' + + def apply(self): + os.makedirs(self._save_dir, exist_ok=True) + for subset_name, subset in self._extractor.subsets().items(): + data = [] + item_ids = {} + image_sizes = {} + for item in subset: + anns = [a.label for a in item.annotations + if a.type == AnnotationType.label] + label = MnistCsvPath.NONE_LABEL + if anns: + label = anns[0] + + if item.has_image and self._save_images: + image = item.image + if not image.has_data: + data.append([label, None]) + else: + if image.data.shape[0] != MnistCsvPath.IMAGE_SIZE or \ + image.data.shape[1] != MnistCsvPath.IMAGE_SIZE: + image_sizes[len(data)] = [image.data.shape[0], + image.data.shape[1]] + image = image.data.reshape(-1).astype(np.uint8).tolist() + image.insert(0, label) + data.append(image) + else: + data.append([label]) + + if item.id != str(len(data) - 1): + item_ids[len(data) - 1] = item.id + + anno_file = osp.join(self._save_dir, 'mnist_%s.csv' % subset_name) + self.save_in_csv(anno_file, data) + + # it is't in the original format, + # this is for storng other names and sizes of images + if len(item_ids) or len(image_sizes): + meta = [] + if len(item_ids) and len(image_sizes): + # other names and sizes of images + size = [MnistCsvPath.IMAGE_SIZE, MnistCsvPath.IMAGE_SIZE] + for i in range(len(data)): + w, h = image_sizes.get(i, size) + meta.append([item_ids.get(i, i), w, h]) + + elif len(item_ids): + # other names of images + for i in range(len(data)): + meta.append([item_ids.get(i, i)]) + + elif len(image_sizes): + # other sizes of images + size = [MnistCsvPath.IMAGE_SIZE, MnistCsvPath.IMAGE_SIZE] + for i in range(len(data)): + meta.append(image_sizes.get(i, size)) + + metafile = osp.join(self._save_dir, 'meta_%s.csv' % subset_name) + self.save_in_csv(metafile, meta) + + self.save_labels() + + def save_in_csv(self, path, data): + with open(path, 'w', encoding='utf-8') as f: + for row in data: + f.write(','.join([str(p) for p in row]) + "\n") + + def save_labels(self): + labels_file = osp.join(self._save_dir, 'labels.txt') + with open(labels_file, 'w', encoding='utf-8') as f: + f.writelines(l.name + '\n' + for l in self._extractor.categories().get( + AnnotationType.label, LabelCategories()) + ) diff --git a/datumaro/plugins/mnist_format.py b/datumaro/plugins/mnist_format.py new file mode 100644 index 0000000000..0cd97b06df --- /dev/null +++ b/datumaro/plugins/mnist_format.py @@ -0,0 +1,209 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import gzip +import os +import os.path as osp + +import numpy as np +from datumaro.components.converter import Converter +from datumaro.components.extractor import (AnnotationType, DatasetItem, + Importer, Label, LabelCategories, SourceExtractor) + + +class MnistPath: + TEST_LABELS_FILE = 't10k-labels-idx1-ubyte.gz' + TEST_IMAGES_FILE = 't10k-images-idx3-ubyte.gz' + LABELS_FILE = '-labels-idx1-ubyte.gz' + IMAGES_FILE = '-images-idx3-ubyte.gz' + IMAGE_SIZE = 28 + NONE_LABEL = 255 + +class MnistExtractor(SourceExtractor): + def __init__(self, path, subset=None): + if not osp.isfile(path): + raise FileNotFoundError("Can't read annotation file '%s'" % path) + + if not subset: + file_name = osp.splitext(osp.basename(path))[0] + if file_name.startswith('t10k'): + subset = 'test' + else: + subset = file_name.split('-', maxsplit=1)[0] + + super().__init__(subset=subset) + self._dataset_dir = osp.dirname(path) + + self._categories = self._load_categories() + + self._items = list(self._load_items(path).values()) + + def _load_categories(self): + label_cat = LabelCategories() + + labels_file = osp.join(self._dataset_dir, 'labels.txt') + if osp.isfile(labels_file): + with open(labels_file, encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + label_cat.add(line) + else: + for i in range(10): + label_cat.add(str(i)) + + return { AnnotationType.label: label_cat } + + def _load_items(self, path): + items = {} + with gzip.open(path, 'rb') as lbpath: + labels = np.frombuffer(lbpath.read(), dtype=np.uint8, offset=8) + + meta = [] + metafile = osp.join(self._dataset_dir, self._subset + '-meta.gz') + if osp.isfile(metafile): + with gzip.open(metafile, 'rb') as f: + meta = np.frombuffer(f.read(), dtype=' +# or +datum create +datum add path -f mnist +``` + +There are two ways to create Datumaro project and add MNIST in CSV dataset to it: + +``` bash +datum import --format mnist_csv --input-path +# or +datum create +datum add path -f mnist_csv +``` + +It is possible to specify project name and project directory run +`datum create --help` for more information. + +MNIST dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── labels.txt # list of non-digit labels (optional) + ├── t10k-images-idx3-ubyte.gz + ├── t10k-labels-idx1-ubyte.gz + ├── train-images-idx3-ubyte.gz + └── train-labels-idx1-ubyte.gz +``` +MNIST in CSV dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── labels.txt # list of non-digit labels (optional) + ├── mnist_test.csv + └── mnist_train.csv +``` +If the dataset needs non-digit labels, you need to add the labels.txt +to the dataset folder. +For example, labels.txt for Fashion MNIST labels contains the following: + +``` +T-shirt/top +Trouser +Pullover +Dress +Coat +Sandal +Shirt +Sneaker +Bag +Ankle boot +``` + +MNIST format only supports single channel 28 x 28 images. + +## Export to other formats + +Datumaro can convert MNIST dataset into any other format [Datumaro supports](../docs/user_manual.md#supported-formats). +To get the expected result, the dataset needs to be converted to formats +that support the classification task (e.g. CIFAR-10/100, ImageNet, PascalVOC, etc.) +There are few ways to convert MNIST dataset to other dataset format: + +``` bash +datum project import -f mnist -i +datum export -f imagenet -o +# or +datum convert -if mnist -i -f imagenet -o +``` + +These commands also work for MNIST in CSV if you use `mnist_csv` instead of `mnist`. + +## Export to MNIST + +There are few ways to convert dataset to MNIST format: + +``` bash +# export dataset into MNIST format from existing project +datum export -p -f mnist -o \ + -- --save-images +# converting to MNIST format from other format +datum convert -if imagenet -i \ + -f mnist -o -- --save-images +``` + +Extra options for export to MNIST format: + +- `--save-images` allow to export dataset with saving images +(by default `False`); +- `--image-ext ` allow to specify image extension +for exporting dataset (by default `.png`). + +These commands also work for MNIST in CSV if you use `mnist_csv` instead of `mnist`. + +## Particular use cases + +Datumaro supports filtering, transformation, merging etc. for all formats +and for the MNIST format in particular. Follow [user manual](../docs/user_manual.md) +to get more information about these operations. + +There are few examples of using Datumaro operations to solve +particular problems with MNIST dataset: + +### Example 1. How to create custom MNIST-like dataset + +```python +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import Label, DatasetItem + +dataset = Dataset.from_iterable([ + DatasetItem(id=0, image=np.ones((28, 28)), + annotations=[Label(2)] + ), + DatasetItem(id=1, image=np.ones((28, 28)), + annotations=[Label(7)] + ) +], categories=[str(label) for label in range(10)]) + +dataset.export('./dataset', format='mnist') +``` + +### Example 2. How to filter and convert MNIST dataset to ImageNet + +Convert MNIST dataset to ImageNet format, keep only images with `3` class presented: + +``` bash +# Download MNIST dataset: +# https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz +# https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz +datum convert --input-format mnist --input-path \ + --output-format imagenet \ + --filter '/item[annotation/label="3"]' +``` diff --git a/docs/user_manual.md b/docs/user_manual.md index 0f1f57b336..550f5e88f5 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -118,6 +118,14 @@ List of supported formats: - CIFAR-10/100 (`classification` (python version)) - [Format specification](https://www.cs.toronto.edu/~kriz/cifar.html) - [Dataset example](../tests/assets/cifar_dataset) +- MNIST (`classification`) + - [Format specification](http://yann.lecun.com/exdb/mnist/) + - [Dataset example](../tests/assets/mnist_dataset) + - [Format documentation](./mnist_user_manual.md) +- MNIST in CSV (`classification`) + - [Format specification](https://pjreddie.com/projects/mnist-in-csv/) + - [Dataset example](../tests/assets/mnist_csv_dataset) + - [Format documentation](./mnist_user_manual.md) - CamVid (`segmentation`) - [Format specification](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [Dataset example](../tests/assets/camvid_dataset) diff --git a/tests/assets/mnist_csv_dataset/mnist_test.csv b/tests/assets/mnist_csv_dataset/mnist_test.csv new file mode 100644 index 0000000000..d07be3847f --- /dev/null +++ b/tests/assets/mnist_csv_dataset/mnist_test.csv @@ -0,0 +1,3 @@ +0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 diff --git a/tests/assets/mnist_csv_dataset/mnist_train.csv b/tests/assets/mnist_csv_dataset/mnist_train.csv new file mode 100644 index 0000000000..c93c7fca93 --- /dev/null +++ b/tests/assets/mnist_csv_dataset/mnist_train.csv @@ -0,0 +1,2 @@ +5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 diff --git a/tests/assets/mnist_dataset/t10k-images-idx3-ubyte.gz b/tests/assets/mnist_dataset/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000000000000000000000000000000000000..811cde6b46f8ac4542e85582d0d1c35155da4aba GIT binary patch literal 74 zcmb2|=HQq%eR2ZR{}MxkY~9S<#Prl+-OQ8ADfgxLCakm2}0|Nl_D;Zq? literal 0 HcmV?d00001 diff --git a/tests/assets/mnist_dataset/t10k-labels-idx1-ubyte.gz b/tests/assets/mnist_dataset/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000000000000000000000000000000000000..a06f7a317c867ae3bdd32dcac470417bce34ce4c GIT binary patch literal 54 zcmb2|=HQq%eR2ZR{}MxkY~7s1q|}^Z-OQ8 Date: Mon, 17 May 2021 11:58:46 +0300 Subject: [PATCH 22/40] make formats docs folder, create COCO format documentation (#241) * Make formats docs folder, move format docs * Create COCO format documentation --- CHANGELOG.md | 1 + docs/formats/coco_user_manual.md | 218 +++++++++++++++++++ docs/{ => formats}/mnist_user_manual.md | 7 +- docs/{ => formats}/pascal_voc_user_manual.md | 8 +- docs/user_manual.md | 7 +- 5 files changed, 232 insertions(+), 9 deletions(-) create mode 100644 docs/formats/coco_user_manual.md rename docs/{ => formats}/mnist_user_manual.md (96%) rename docs/{ => formats}/pascal_voc_user_manual.md (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index d177cf58a2..3e189f7b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support COCO panoptic and stuff format () - Documentation file and integration tests for Pascal VOC format () - Support for MNIST and MNIST in CSV dataset formats () +- Documentation file for COCO format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/docs/formats/coco_user_manual.md b/docs/formats/coco_user_manual.md new file mode 100644 index 0000000000..2f824dd11a --- /dev/null +++ b/docs/formats/coco_user_manual.md @@ -0,0 +1,218 @@ +# COCO user manual + +## Contents + +- [Format specification](#format-specification) +- [Load COCO dataset](#load-COCO-dataset) +- [Export to other formats](#export-to-other-formats) +- [Export to COCO](#export-to-COCO) +- [Particular use cases](#particular-use-cases) + +## Format specification + +COCO format specification available [here](https://cocodataset.org/#format-data). + +COCO dataset format supports `captions`, `image_info`, `instances`, `panoptic`, +`person_keypoints`, `stuff` annotation tasks +and, as Datumaro extension, `label` (like `instances` with only `category_id`) + +## Load COCO dataset + +The COCO dataset is available for free download: + +Images: +- [train images](http://images.cocodataset.org/zips/train2017.zip) +- [val images](http://images.cocodataset.org/zips/val2017.zip) +- [test images](http://images.cocodataset.org/zips/test2017.zip) +- [unlabeled images](http://images.cocodataset.org/zips/unlabeled2017.zip) + +Annotations: +- [captions](http://images.cocodataset.org/annotations/annotations_trainval2017.zip) +- [image_info](http://images.cocodataset.org/annotations/image_info_test2017.zip) +- [instances](http://images.cocodataset.org/annotations/annotations_trainval2017.zip) +- [panoptic](http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip) +- [person_keypoints](http://images.cocodataset.org/annotations/annotations_trainval2017.zip) +- [stuff](http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip) + +There are two ways to create Datumaro project and add COCO dataset to it: + +``` bash +datum import --format coco --input-path +# or +datum create +datum add path -f coco +``` + +It is possible to specify project name and project directory run +`datum create --help` for more information. + +COCO dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── images/ + │ ├── train + │ │ ├── + │ │ ├── + │ │ └── ... + │ ├── val + │ │ ├── + │ │ ├── + │ │ └── ... + ├── annotations/ + │ └── _train.json + │ └── _test.json +``` + +For `panoptic` COCO dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── images/ + │ ├── train + │ │ ├── + │ │ ├── + │ │ └── ... + │ ├── val + │ │ ├── + │ │ ├── + │ │ └── ... + ├── annotations/ + │ ├── panoptic_train + │ │ ├── + │ │ ├── + │ │ └── ... + │ ├── panoptic_train.json + │ ├── panoptic_val + │ │ ├── + │ │ ├── + │ │ └── ... + │ └── panoptic_val.json +``` + +You can import dataset for specific tasks +of COCO dataset instead of the whole dataset, +for example: + +``` bash +datum import --format coco_stuff --input-path +``` + +Datumaro supports the following COCO tasks: +- [Image Captioning](https://cocodataset.org/#captions-2015) (`coco_caption`) +- [Object Detection](https://cocodataset.org/#detection-2020) (`coco_instances`) +- Image classification (our extension) (`coco_labels`) - a format like Object Detection, which uses + only `category_id` and `score` annotation fields +- [Panoptic Segmentation](https://cocodataset.org/#panoptic-2020) (`coco_panoptic`) +- [Keypoint Detection](https://cocodataset.org/#keypoints-2020) (`coco_person_keypoints`) +- [Stuff Segmentation](https://cocodataset.org/#stuff-2019) (`coco_stuff`) + +To make sure that the selected dataset has been added to the project, you can run +`datum info`, which will display the project and dataset information. + +## Export to other formats + +Datumaro can convert COCO dataset into any other format [Datumaro supports](../user_manual.md#supported-formats). +To get the expected result, the dataset needs to be converted to formats +that support the specified task (e.g. for panoptic segmentation - VOC, CamVID) +There are few ways to convert COCO dataset to other dataset format: + +``` bash +datum project import -f coco -i +datum export -f voc -o +# or +datum convert -if coco -i -f voc -o +``` + +Some formats provide extra options for conversion. +These options are passed after double dash (`--`) in the command line. +To get information about them, run + +`datum export -f -- -h` + +## Export to COCO + +There are few ways to convert dataset to COCO format: + +``` bash +# export dataset into COCO format from existing project +datum export -p -f coco -o \ + -- --save-images +# converting to COCO format from other format +datum convert -if voc -i \ + -f coco -o -- --save-images +``` + +Extra options for export to COCO format: +- `--save-images` allow to export dataset with saving images +(by default `False`); +- `--image-ext IMAGE_EXT` allow to specify image extension +for exporting dataset (by default - keep original or use `.jpg`, if none); +- `--segmentation-mode MODE` allow to specify save mode for instance segmentation: + - 'guess': guess the mode for each instance (using 'is_crowd' attribute as hint) + - 'polygons': save polygons( merge and convert masks, prefer polygons) + - 'mask': save masks (merge and convert polygons, prefer masks) +(by default `guess`); +- `--crop-covered` allow to crop covered segments so that background objects +segmentation was more accurate (by default `False`); +- `--allow-attributes ALLOW_ATTRIBUTES` allow export of attributes +(by default `True`); +- `--reindex REINDEX` allow to assign new indices to images and annotations, +useful to avoid merge conflicts (by default `False`); +- `--merge-images` allow to save all images into a single directory +(by default `False`); +- `--tasks TASKS` allow to specify tasks for export dataset, +by default Datumaro uses all tasks. Example: + +```bash +datum import -o project -f coco -i +datum export -p project -f coco -- --tasks instances,stuff +``` + +## Particular use cases + +Datumaro supports filtering, transformation, merging etc. for all formats +and for the COCO format in particular. Follow +[user manual](../user_manual.md) +to get more information about these operations. + +There are few examples of using Datumaro operations to solve +particular problems with COCO dataset: + +### Example 1. How to load an original panoptic COCO dataset ans convert to Pascal VOC + +```bash +datum create -o project +datum add path -p project -f coco_panoptic ./COCO/annotations/panoptic_val2017.json +datum stats -p project +datum export -p final_project -o dataset -f voc --overwrite -- --save-images +``` + +### Example 2. How to create custom COCO-like dataset + +```python +import numpy as np +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import Mask, DatasetItem + +dataset = Dataset.from_iterable([ + DatasetItem(id='000000000001', + image=np.ones((1, 5, 3)), + subset='val', + attributes={'id': 40}, + annotations=[ + Mask(image=np.array([[0, 0, 1, 1, 0]]), label=3, + id=7, group=7, attributes={'is_crowd': False}), + Mask(image=np.array([[0, 1, 0, 0, 1]]), label=1, + id=20, group=20, attributes={'is_crowd': True}), + ] + ), + ], categories=['a', 'b', 'c', 'd']) + +dataset.export('./dataset', format='coco_panoptic') +``` + +More examples of working with COCO dataset from code can be found in +[tests](../../tests/test_coco_format.py) diff --git a/docs/mnist_user_manual.md b/docs/formats/mnist_user_manual.md similarity index 96% rename from docs/mnist_user_manual.md rename to docs/formats/mnist_user_manual.md index 77ad1e50c8..01645e2827 100644 --- a/docs/mnist_user_manual.md +++ b/docs/formats/mnist_user_manual.md @@ -99,7 +99,7 @@ MNIST format only supports single channel 28 x 28 images. ## Export to other formats -Datumaro can convert MNIST dataset into any other format [Datumaro supports](../docs/user_manual.md#supported-formats). +Datumaro can convert MNIST dataset into any other format [Datumaro supports](../user_manual.md#supported-formats). To get the expected result, the dataset needs to be converted to formats that support the classification task (e.g. CIFAR-10/100, ImageNet, PascalVOC, etc.) There are few ways to convert MNIST dataset to other dataset format: @@ -138,7 +138,7 @@ These commands also work for MNIST in CSV if you use `mnist_csv` instead of `mni ## Particular use cases Datumaro supports filtering, transformation, merging etc. for all formats -and for the MNIST format in particular. Follow [user manual](../docs/user_manual.md) +and for the MNIST format in particular. Follow [user manual](../user_manual.md) to get more information about these operations. There are few examples of using Datumaro operations to solve @@ -174,3 +174,6 @@ datum convert --input-format mnist --input-path \ --output-format imagenet \ --filter '/item[annotation/label="3"]' ``` + +More examples of working with MNIST dataset from code can be found in +[tests_mnist](../../tests/test_mnist_format.py) and [tests_mnist_csv](../../tests/test_mnist_csv_format.py) diff --git a/docs/pascal_voc_user_manual.md b/docs/formats/pascal_voc_user_manual.md similarity index 98% rename from docs/pascal_voc_user_manual.md rename to docs/formats/pascal_voc_user_manual.md index 0e8244f44b..0e5db0e925 100644 --- a/docs/pascal_voc_user_manual.md +++ b/docs/formats/pascal_voc_user_manual.md @@ -136,7 +136,7 @@ To make sure that the selected dataset has been added to the project, you can ru ## Export to other formats Datumaro can convert Pascal VOC dataset into any other format -[Datumaro supports](../docs/user_manual.md#supported-formats). +[Datumaro supports](../user_manual.md#supported-formats). Such conversion will only be successful if the output format can represent the type of dataset you want to convert, @@ -211,7 +211,7 @@ datum export -f voc_segmentation -- --label-map voc Datumaro supports filtering, transformation, merging etc. for all formats and for the Pascal VOC format in particular. Follow -[user manual](../docs/user_manual.md) +[user manual](../user_manual.md) to get more information about these operations. There are few examples of using Datumaro operations to solve @@ -313,5 +313,5 @@ After executing this code, we can see that there are 5826 images in Pascal VOC 2012 has for segmentation task and this result is the same as the [official documentation](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/dbstats.html) -Some examples of working with Pascal VOC dataset from code you can found in -[tests](../tests/test_voc_format.py) \ No newline at end of file +More examples of working with Pascal VOC dataset from code can be found in +[tests](../../tests/test_voc_format.py) diff --git a/docs/user_manual.md b/docs/user_manual.md index 550f5e88f5..1f32666aeb 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -89,10 +89,11 @@ List of supported formats: - [Format specification](http://cocodataset.org/#format-data) - [Dataset example](../tests/assets/coco_dataset) - `labels` are our extension - like `instances` with only `category_id` + - [Format documentation](./formats/coco_user_manual.md) - PASCAL VOC (`classification`, `detection`, `segmentation` (class, instances), `action_classification`, `person_layout`) - [Format specification](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/htmldoc/index.html) - [Dataset example](../tests/assets/voc_dataset) - - [Format documentation](./pascal_voc_user_manual.md) + - [Format documentation](./formats/pascal_voc_user_manual.md) - YOLO (`bboxes`) - [Format specification](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) - [Dataset example](../tests/assets/yolo_dataset) @@ -121,11 +122,11 @@ List of supported formats: - MNIST (`classification`) - [Format specification](http://yann.lecun.com/exdb/mnist/) - [Dataset example](../tests/assets/mnist_dataset) - - [Format documentation](./mnist_user_manual.md) + - [Format documentation](./formats/mnist_user_manual.md) - MNIST in CSV (`classification`) - [Format specification](https://pjreddie.com/projects/mnist-in-csv/) - [Dataset example](../tests/assets/mnist_csv_dataset) - - [Format documentation](./mnist_user_manual.md) + - [Format documentation](./formats/mnist_user_manual.md) - CamVid (`segmentation`) - [Format specification](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [Dataset example](../tests/assets/camvid_dataset) From 8c7bbc52852373ee96f858a537fd75fd0f915ada Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Mon, 17 May 2021 11:59:34 +0300 Subject: [PATCH 23/40] Fixes in CIFAR dataset format (#243) * Add folder creation * Update changelog --- CHANGELOG.md | 2 +- datumaro/plugins/cifar_format.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e189f7b72..6a6900aa8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Support for escaping in attribiute values in LabelMe format () - Support for Segmentation Splitting () -- Support for CIFAR-10/100 dataset format () +- Support for CIFAR-10/100 dataset format (, ) - Support COCO panoptic and stuff format () - Documentation file and integration tests for Pascal VOC format () - Support for MNIST and MNIST in CSV dataset formats () diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index 2a74ba99c2..287f00de8e 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +import os import os.path as osp import pickle @@ -117,8 +118,9 @@ class CifarConverter(Converter): DEFAULT_IMAGE_EXT = '.png' def apply(self): - label_categories = self._extractor.categories()[AnnotationType.label] + os.makedirs(self._save_dir, exist_ok=True) + label_categories = self._extractor.categories()[AnnotationType.label] label_names = [] for label in label_categories: label_names.append(label.name) From 6782295f59714b2188e6fbe8e900a24dec7cbfd5 Mon Sep 17 00:00:00 2001 From: Kirill Sizov Date: Thu, 20 May 2021 15:23:41 +0300 Subject: [PATCH 24/40] Add user documentation file and integration tests for YOLO format (#246) * add user documentation file for yolo * add integraion tests * update user manual * update changelog --- CHANGELOG.md | 1 + docs/formats/yolo_user_manual.md | 210 ++++++++++++++++++ docs/user_manual.md | 1 + .../voc_dataset1/JPEGImages/2007_000001.jpg | Bin 0 -> 336 bytes tests/cli/test_yolo_format.py | 157 +++++++++++++ 5 files changed, 369 insertions(+) create mode 100644 docs/formats/yolo_user_manual.md create mode 100644 tests/assets/voc_dataset/voc_dataset1/JPEGImages/2007_000001.jpg create mode 100644 tests/cli/test_yolo_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a6900aa8e..a256c4877c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Documentation file and integration tests for Pascal VOC format () - Support for MNIST and MNIST in CSV dataset formats () - Documentation file for COCO format () +- Documentation file and integration tests for YOLO format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/docs/formats/yolo_user_manual.md b/docs/formats/yolo_user_manual.md new file mode 100644 index 0000000000..266547945f --- /dev/null +++ b/docs/formats/yolo_user_manual.md @@ -0,0 +1,210 @@ +# YOLO user manual + +## Contents +- [Format specification](#format-specification) +- [Load YOLO dataset](#load-yolo-dataset) +- [Export to other formats](#export-to-other-formats) +- [Export to YOLO format](#export-to-yolo-format) +- [Particular use cases](#particular-use-cases) + +## Format specification + +- The YOLO dataset format is for training and validating object detection models. +Specification for this format available +[here](https://github.com/AlexeyAB/darknet#how-to-train-to-detect-your-custom-objects). +And also you can find some official examples on working with YOLO dataset +[here](https://pjreddie.com/darknet/yolo/); + +- The YOLO dataset format support the following types of annotations: + - `Bounding boxes` + +- YOLO format doesn't support attributes for annotations; + +- The format only supports subsets named `train` or `valid`. + +## Load YOLO dataset + +Few ways to create Datumaro project and add YOLO dataset to it: + +```bash +datum import -o project -f yolo -i + +# another way to do the same: +datum create -o project +datum add path -p project -f yolo -i + +# and you can add another one yolo dataset: +datum add path -p project -f yolo -i +``` + +YOLO dataset directory should have the following structure: + + +``` +└─ yolo_dataset/ + │ + ├── obj.names # file with list of classes + ├── obj.data # file with dataset information + ├── train.txt # list of image paths in train subset + ├── valid.txt # list of image paths in valid subset + │ + ├── obj_train_data/ # directory with annotations and images for train subset + │ ├── image1.txt # list of labeled bounding boxes for image1 + │ ├── image1.jpg + │ ├── image2.txt + │ ├── image2.jpg + │ ├── ... + │ + ├── obj_valid_data/ # directory with annotations and images for valid subset + │ ├── image101.txt + │ ├── image101.jpg + │ ├── image102.txt + │ ├── image102.jpg + │ ├── ... +``` +> YOLO dataset cannot contain a subset with a name other than `train` or `valid`. +If imported dataset contains such subsets, they will be ignored. +If you are exporting a project into yolo format, +all subsets different from `train` and `valid` will be skipped. +If there is no subset separation in a project, the data +will be saved in `train` subset. + +- `obj.data` should have the following content, it is not necessary to have both +subsets, but necessary to have one of them: +``` +classes = 5 # optional +names = +train = +valid = +backup = backup/ # optional +``` +- `obj.names` contain list of classes. +The line number for the class is the same as its index: +``` +label1 # label1 has index 0 +label2 # label2 has index 1 +label3 # label2 has index 2 +... +``` +- Files `train.txt` and `valid.txt` should have the following structure: +``` + + +... +``` +- Files in directories `obj_train_data/` and `obj_valid_data/` +should contain information about labeled bounding boxes +for images: +``` +# image1.txt: +# +0 0.250000 0.400000 0.300000 0.400000 +3 0.600000 0.400000 0.400000 0.266667 +``` +Here `x`, `y`, `width`, and `height` are relative to the image's width and height. + +## Export to other formats + +Datumaro can convert YOLO dataset into any other format +[Datumaro supports](../docs/user_manual.md#supported-formats). +For successful conversion the output format should support +object detection task (e.g. Pascal VOC, COCO, TF Detection API etc.) + +Examples: +```bash +datum import -o project -f yolo -i +datum export -p project -f voc -o +``` + +```bash +datum convert -if yolo -i \ + -f coco_instances -o +``` + +## Export to YOLO format + +Datumaro can convert an existing dataset to YOLO format, +if the dataset supports object detection task. + +Example: + +``` +datum import -p project -f coco_instances -i +datum export -p project -f yolo -o -- --save-images +``` + +Extra options for export to YOLO format: + +- `--save-images` allow to export dataset with saving images +(default: `False`); +- `--image-ext ` allow to specify image extension +for exporting dataset (default: use original or `.jpg`, if none). + +## Particular use cases + +### How to prepare PASCAL VOC dataset for exporting to YOLO format dataset? + +```bash +datum import -o project -f voc -i ./VOC2012 +datum filter -p project -e '/item[subset="train" or subset="val"]' -o trainval_voc +datum transform -p trainval_voc -o trainvalid_voc \ + -t map_subsets -- -s train:train -s val:valid +datum export -p trainvalid_voc -f yolo -o ./yolo_dataset -- --save-images +``` + +### How to remove some class from YOLO dataset? +Delete all items, which contain `cat` objects and remove +`cat` from list of classes: +```bash +datum import -o project -f yolo -i ./yolo_dataset +datum filter -p project -o filtered -m i+a -e '/item/annotation[label!="cat"]' +datum transform -p filtered -o without_cat -t remap_labels -- -l cat: +datum export -p without_cat -f yolo -o ./yolo_without_cats +``` + +### How to create custom dataset in YOLO format? +```python +import numpy as np +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import Bbox, DatasetItem + +dataset = Dataset.from_iterable([ + DatasetItem(id='image_001', subset='train', + image=np.ones((20, 20, 3)), + annotations=[ + Bbox(3.0, 1.0, 8.0, 5.0, label=1), + Bbox(1.0, 1.0, 10.0, 1.0, label=2) + ] + ), + DatasetItem(id='image_002', subset='train', + image=np.ones((15, 10, 3)), + annotations=[ + Bbox(4.0, 4.0, 4.0, 4.0, label=3) + ] + ) +], categories=['house', 'bridge', 'crosswalk', 'traffic_light']) + +dataset.export('../yolo_dataset', format='yolo', save_images=True) +``` + +### How to get information about objects on each images? + +If you only want information about label names for each +images, then you can get it from code: +```python +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import AnnotationType + +dataset = Dataset.import_from('./yolo_dataset', format='yolo') +cats = dataset.categories()[AnnotationType.label] + +for item in dataset: + for ann in item.annotations: + print(item.id, cats[ann.label].name) +``` + +And If you want complete information about each items you can run: +```bash +datum import -o project -f yolo -i ./yolo_dataset +datum filter -p project --dry-run -e '/item' +``` \ No newline at end of file diff --git a/docs/user_manual.md b/docs/user_manual.md index 1f32666aeb..5e5a5e22c0 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -97,6 +97,7 @@ List of supported formats: - YOLO (`bboxes`) - [Format specification](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) - [Dataset example](../tests/assets/yolo_dataset) + - [Format documentation](./formats/yolo_user_manual.md) - TF Detection API (`bboxes`, `masks`) - Format specifications: [bboxes](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/using_your_own_dataset.md), [masks](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/instance_segmentation.md) - [Dataset example](../tests/assets/tf_detection_api_dataset) diff --git a/tests/assets/voc_dataset/voc_dataset1/JPEGImages/2007_000001.jpg b/tests/assets/voc_dataset/voc_dataset1/JPEGImages/2007_000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6c07340b733a490751136e26c942f2e58a73794d GIT binary patch literal 336 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<Mn*w~ z|3?_)frhg(f&l{*FfuW-u(GjpaB^`26>Jq?U}9uuW@2GxWo2Ojs;&jfGq4D<3Mm>o zvIz$!vMUve7&T5@$f4}C@t|nX#SbdRNkvVZTw>x9l2WQ_>Kd9_CZ=ZQ7M51dF0O9w z9-dyoA)#U65s^{JDXD4c8JStdC8cHM6_r)ZEv;?s9i3g1CQq3GGAU*RJ2VdF$b$$4{OPfBE|D a`;VW${@-HY00o;p!v`*nMP1teZvp^yfopjH literal 0 HcmV?d00001 diff --git a/tests/cli/test_yolo_format.py b/tests/cli/test_yolo_format.py new file mode 100644 index 0000000000..d5242daefa --- /dev/null +++ b/tests/cli/test_yolo_format.py @@ -0,0 +1,157 @@ +import numpy as np +import os.path as osp + +from unittest import TestCase + +from datumaro.cli.__main__ import main +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import (DatasetItem, + AnnotationType, Bbox) +from datumaro.util.test_utils import TestDir, compare_datasets +import datumaro.plugins.voc_format.format as VOC + +def run(test, *args, expected_code=0): + test.assertEqual(expected_code, main(args), str(args)) + +class YoloIntegrationScenarios(TestCase): + def test_can_save_and_load_yolo_dataset(self): + target_dataset = Dataset.from_iterable([ + DatasetItem(id='1', subset='train', + image=np.ones((10, 15, 3)), + annotations=[ + Bbox(3.0, 3.0, 2.0, 3.0, label=4), + Bbox(0.0, 2.0, 4.0, 2.0, label=2) + ] + ) + ], categories=['label_' + str(i) for i in range(10)]) + + with TestDir() as test_dir: + yolo_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'yolo_dataset') + + run(self, 'import', '-o', test_dir, '-f', 'yolo', '-i', yolo_dir) + + export_dir = osp.join(test_dir, 'export_dir') + run(self, 'export', '-p', test_dir, '-o', export_dir, + '-f', 'yolo', '--', '--save-images') + + parsed_dataset = Dataset.import_from(export_dir, format='yolo') + compare_datasets(self, target_dataset, parsed_dataset) + + def test_can_export_mot_as_yolo(self): + target_dataset = Dataset.from_iterable([ + DatasetItem(id='1', subset='train', + annotations=[ + Bbox(0.0, 4.0, 4.0, 8.0, label=2) + ] + ) + ], categories=['label_' + str(i) for i in range(10)]) + + with TestDir() as test_dir: + mot_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'mot_dataset') + + run(self, 'create', '-o', test_dir) + run(self, 'add', 'path', '-p', test_dir, '-f', 'mot_seq', mot_dir) + + yolo_dir = osp.join(test_dir, 'yolo_dir') + run(self, 'export', '-p', test_dir, '-o', yolo_dir, + '-f', 'yolo', '--', '--save-images') + + parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') + compare_datasets(self, target_dataset, parsed_dataset) + + def test_can_convert_voc_to_yolo(self): + target_dataset = Dataset.from_iterable([ + DatasetItem(id='2007_000001', subset='train', + annotations=[ + Bbox(8.0, 2.5, 4.0, 1.0, label=15), + Bbox(2.0, 1.0, 4.0, 1.0, label=8), + Bbox(11.0, 3.0, 4.0, 1.0, label=22) + ] + ) + ], categories=[label.name for label in + VOC.make_voc_categories()[AnnotationType.label]]) + + with TestDir() as test_dir: + voc_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'voc_dataset', 'voc_dataset1') + yolo_dir = osp.join(test_dir, 'yolo_dir') + + run(self, 'convert', '-if', 'voc', '-i', voc_dir, + '-f', 'yolo', '-o', yolo_dir, '--', '--save-images') + + parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') + compare_datasets(self, target_dataset, parsed_dataset) + + def test_can_ignore_non_supported_subsets(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='img1', subset='test', + image=np.ones((10, 20, 3)), + annotations=[ + Bbox(1.0, 2.0, 1.0, 1.0, label=0) + ] + ), + DatasetItem(id='img2', subset='train', + image=np.ones((10, 5, 3)), + annotations=[ + Bbox(3.0, 1.0, 2.0, 1.0, label=1) + ] + ) + ], categories=[str(i) for i in range(4)]) + + target_dataset = Dataset.from_iterable([ + DatasetItem(id='img2', subset='train', + image=np.ones((10, 5, 3)), + annotations=[ + Bbox(3.0, 1.0, 2.0, 1.0, label=1) + ] + ) + ], categories=[str(i) for i in range(4)]) + + with TestDir() as test_dir: + dataset_dir = osp.join(test_dir, 'dataset_dir') + source_dataset.save(dataset_dir, save_images=True) + + run(self, 'create', '-o', test_dir) + run(self, 'add', 'path', '-p', test_dir, '-f', 'datumaro', dataset_dir) + + yolo_dir = osp.join(test_dir, 'yolo_dir') + run(self, 'export', '-p', test_dir, '-o', yolo_dir, + '-f', 'yolo', '--', '--save-images') + + parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') + compare_datasets(self, target_dataset, parsed_dataset) + + def test_can_delete_labels_from_yolo_dataset(self): + target_dataset = Dataset.from_iterable([ + DatasetItem(id='1', subset='train', + image=np.ones((10, 15, 3)), + annotations=[ + Bbox(0.0, 2.0, 4.0, 2.0, label=0) + ] + ) + ], categories=['label_2']) + + with TestDir() as test_dir: + yolo_dir = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], + 'tests', 'assets', 'yolo_dataset') + + run(self, 'create', '-o', test_dir) + run(self, 'add', 'path', '-p', test_dir, '-f', 'yolo', yolo_dir) + + filtered_path = osp.join(test_dir, 'filtered') + run(self, 'filter', '-p', test_dir, '-o', filtered_path, + '-m', 'i+a', '-e', "/item/annotation[label='label_2']") + + result_path = osp.join(test_dir, 'result') + run(self, 'transform', '-p', filtered_path, '-o', result_path, + '-t', 'remap_labels', '--', '-l', 'label_2:label_2', + '--default', 'delete') + + export_dir = osp.join(test_dir, 'export') + run(self, 'export', '-p', result_path, '-o', export_dir, + '-f', 'yolo', '--', '--save-image') + + parsed_dataset = Dataset.import_from(export_dir, format='yolo') + compare_datasets(self, target_dataset, parsed_dataset) \ No newline at end of file From 58892deef4165d20646f15c07975824c9061d612 Mon Sep 17 00:00:00 2001 From: Zoya Maslova Date: Tue, 25 May 2021 16:16:31 +0300 Subject: [PATCH 25/40] Add Cityscapes format (#249) * add cityscapes format * add format docs * update changelog --- CHANGELOG.md | 3 +- README.md | 1 + datumaro/components/extractor.py | 15 +- datumaro/plugins/cityscapes_format.py | 357 ++++++++++++++++++ datumaro/util/image.py | 10 +- docs/design.md | 2 +- docs/formats/cityscapes_user_manual.md | 176 +++++++++ docs/user_manual.md | 4 + ...tcity_000001_000031_gtFine_instanceIds.png | Bin 0 -> 76 bytes ...tcity_000001_000032_gtFine_instanceIds.png | Bin 0 -> 76 bytes ...tcity_000002_000045_gtFine_instanceIds.png | Bin 0 -> 76 bytes ...tcity_000001_000019_gtFine_instanceIds.png | Bin 0 -> 76 bytes .../defaultcity_000001_000031_leftImg8bit.png | Bin 0 -> 70 bytes .../defaultcity_000001_000032_leftImg8bit.png | Bin 0 -> 70 bytes .../defaultcity_000002_000045_leftImg8bit.png | Bin 0 -> 70 bytes .../defaultcity_000001_000019_leftImg8bit.png | Bin 0 -> 70 bytes tests/test_cityscapes_format.py | 350 +++++++++++++++++ 17 files changed, 909 insertions(+), 9 deletions(-) create mode 100644 datumaro/plugins/cityscapes_format.py create mode 100644 docs/formats/cityscapes_user_manual.md create mode 100644 tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000031_gtFine_instanceIds.png create mode 100644 tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000032_gtFine_instanceIds.png create mode 100644 tests/assets/cityscapes_dataset/gtFine/train/defaultcity/defaultcity_000002_000045_gtFine_instanceIds.png create mode 100644 tests/assets/cityscapes_dataset/gtFine/val/defaultcity/defaultcity_000001_000019_gtFine_instanceIds.png create mode 100644 tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000031_leftImg8bit.png create mode 100644 tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000032_leftImg8bit.png create mode 100644 tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/train/defaultcity/defaultcity_000002_000045_leftImg8bit.png create mode 100644 tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/val/defaultcity/defaultcity_000001_000019_leftImg8bit.png create mode 100644 tests/test_cityscapes_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a256c4877c..e85315d9d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for escaping in attribiute values in LabelMe format () - Support for Segmentation Splitting () - Support for CIFAR-10/100 dataset format (, ) -- Support COCO panoptic and stuff format () +- Support for COCO panoptic and stuff format () - Documentation file and integration tests for Pascal VOC format () - Support for MNIST and MNIST in CSV dataset formats () - Documentation file for COCO format () - Documentation file and integration tests for YOLO format () +- Support for Cityscapes dataset format () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/README.md b/README.md index 7bd9d6252b..2c5957f522 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ CVAT annotations ---> Publication, statistics etc. - [MNIST](http://yann.lecun.com/exdb/mnist/) (`classification`) - [MNIST in CSV](https://pjreddie.com/projects/mnist-in-csv/) (`classification`) - [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) + - [Cityscapes](https://www.cityscapes-dataset.com/) - [CVAT](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) - [LabelMe](http://labelme.csail.mit.edu/Release3.0) - [ICDAR13/15](https://rrc.cvc.uab.es/?ch=2) (`word_recognition`, `text_localization`, `text_segmentation`) diff --git a/datumaro/components/extractor.py b/datumaro/components/extractor.py index b913dece13..ebeaf01ecc 100644 --- a/datumaro/components/extractor.py +++ b/datumaro/components/extractor.py @@ -7,6 +7,7 @@ from glob import iglob from typing import Iterable, List, Dict, Optional import numpy as np +import os import os.path as osp import attr @@ -236,7 +237,7 @@ def __eq__(self, other): class CompiledMask: @staticmethod def from_instance_masks(instance_masks, - instance_ids=None, instance_labels=None): + instance_ids=None, instance_labels=None, dtype=None): from datumaro.util.mask_tools import make_index_mask if instance_ids is not None: @@ -266,7 +267,7 @@ def from_instance_masks(instance_masks, m, idx, instance_id, class_id = next(it) if not class_id: idx = 0 - index_mask = make_index_mask(m, idx) + index_mask = make_index_mask(m, idx, dtype=dtype) instance_map.append(instance_id) class_map.append(class_id) @@ -282,8 +283,8 @@ def from_instance_masks(instance_masks, else: merged_instance_mask = np.array(instance_map, dtype=np.min_scalar_type(instance_map))[index_mask] - merged_class_mask = np.array(class_map, - dtype=np.min_scalar_type(class_map))[index_mask] + dtype_mask = dtype if dtype else np.min_scalar_type(class_map) + merged_class_mask = np.array(class_map, dtype=dtype_mask)[index_mask] return __class__(class_mask=merged_class_mask, instance_mask=merged_instance_mask) @@ -673,7 +674,11 @@ def __call__(self, path, **extra_params): @classmethod def _find_sources_recursive(cls, path, ext, extractor_name, filename='*', dirname='', file_filter=None, max_depth=3): - if path.endswith(ext) and osp.isfile(path): + + if (path.endswith(ext) and osp.isfile(path)) or \ + (not ext and osp.isdir(path) and dirname and \ + os.sep + osp.normpath(dirname) + os.sep in \ + osp.abspath(path) + os.sep): sources = [{'url': path, 'format': extractor_name}] else: sources = [] diff --git a/datumaro/plugins/cityscapes_format.py b/datumaro/plugins/cityscapes_format.py new file mode 100644 index 0000000000..34aca8bd1c --- /dev/null +++ b/datumaro/plugins/cityscapes_format.py @@ -0,0 +1,357 @@ + +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import logging as log +import os +import os.path as osp +from collections import OrderedDict +from enum import Enum +from glob import iglob + +import numpy as np + +from datumaro.components.converter import Converter +from datumaro.components.extractor import (AnnotationType, CompiledMask, + DatasetItem, Importer, LabelCategories, Mask, + MaskCategories, SourceExtractor) +from datumaro.util import str_to_bool +from datumaro.util.annotation_util import make_label_id_mapping +from datumaro.util.image import save_image, load_image +from datumaro.util.mask_tools import generate_colormap, paint_mask + + +CityscapesLabelMap = OrderedDict([ + ('unlabeled', (0, 0, 0)), + ('egovehicle', (0, 0, 0)), + ('rectificationborder', (0, 0, 0)), + ('outofroi', (0, 0, 0)), + ('static', (0, 0, 0)), + ('dynamic', (111, 74, 0)), + ('ground', (81, 0, 81)), + ('road', (128, 64, 128)), + ('sidewalk', (244, 35, 232)), + ('parking', (250, 170, 160)), + ('railtrack', (230, 150, 140)), + ('building', (70, 70, 70)), + ('wall', (102, 102, 156)), + ('fence', (190, 153, 153)), + ('guardrail', (180, 165, 180)), + ('bridge', (150, 100, 100)), + ('tunnel', (150, 120, 90)), + ('pole', (153, 153, 153)), + ('polegroup', (153, 153, 153)), + ('trafficlight', (250, 170, 30)), + ('trafficsign', (220, 220, 0)), + ('vegetation', (107, 142, 35)), + ('terrain', (152, 251, 152)), + ('sky', (70, 130, 180)), + ('person', (220, 20, 60)), + ('rider', (255, 0, 0)), + ('car', (0, 0, 142)), + ('truck', (0, 0, 70)), + ('bus', (0, 60, 100)), + ('caravan', (0, 0, 90)), + ('trailer', (0, 0, 110)), + ('train', (0, 80, 100)), + ('motorcycle', (0, 0, 230)), + ('bicycle', (119, 11, 32)), + ('licenseplate', (0, 0, 142)), +]) + +class CityscapesPath: + GT_FINE_DIR = 'gtFine' + IMGS_FINE_DIR = 'imgsFine' + ORIGINAL_IMAGE_DIR = 'leftImg8bit' + ORIGINAL_IMAGE = '_%s.png' % ORIGINAL_IMAGE_DIR + INSTANCES_IMAGE = '_instanceIds.png' + COLOR_IMAGE = '_color.png' + LABELIDS_IMAGE = '_labelIds.png' + + LABELMAP_FILE = 'label_colors.txt' + +def make_cityscapes_categories(label_map=None): + if label_map is None: + label_map = CityscapesLabelMap + + categories = {} + label_categories = LabelCategories() + for label in label_map: + label_categories.add(label) + categories[AnnotationType.label] = label_categories + + has_colors = any(v is not None for v in label_map.values()) + if not has_colors: # generate new colors + colormap = generate_colormap(len(label_map)) + else: # only copy defined colors + label_id = lambda label: label_categories.find(label)[0] + colormap = { label_id(name): (desc[0], desc[1], desc[2]) + for name, desc in label_map.items() } + mask_categories = MaskCategories(colormap) + mask_categories.inverse_colormap # pylint: disable=pointless-statement + categories[AnnotationType.mask] = mask_categories + return categories + +def parse_label_map(path): + if not path: + return None + + label_map = OrderedDict() + with open(path, 'r') as f: + for line in f: + # skip empty and commented lines + line = line.strip() + if not line or line and line[0] == '#': + continue + + # color, name + label_desc = line.strip().split() + + if 2 < len(label_desc): + name = label_desc[3] + color = tuple([int(c) for c in label_desc[:-1]]) + else: + name = label_desc[0] + color = None + + if name in label_map: + raise ValueError("Label '%s' is already defined" % name) + + label_map[name] = color + return label_map + +def write_label_map(path, label_map): + with open(path, 'w') as f: + for label_name, label_desc in label_map.items(): + if label_desc: + color_rgb = ' '.join(str(c) for c in label_desc) + else: + color_rgb = '' + f.write('%s %s\n' % (color_rgb, label_name)) + +class CityscapesExtractor(SourceExtractor): + def __init__(self, path, subset=None): + assert osp.isdir(path), path + self._path = path + + if not subset: + subset = osp.splitext(osp.basename(path))[0] + self._subset = subset + super().__init__(subset=subset) + + self._categories = self._load_categories(osp.join(self._path, '../../../')) + self._items = list(self._load_items().values()) + + def _load_categories(self, path): + label_map = None + label_map_path = osp.join(path, CityscapesPath.LABELMAP_FILE) + if osp.isfile(label_map_path): + label_map = parse_label_map(label_map_path) + else: + label_map = CityscapesLabelMap + self._labels = [label for label in label_map] + return make_cityscapes_categories(label_map) + + def _load_items(self): + items = {} + annotations_path = osp.normpath(osp.join(self._path, '../../../', + CityscapesPath.GT_FINE_DIR, self._subset)) + + for image_path in iglob( + osp.join(self._path, '**', '*' + CityscapesPath.ORIGINAL_IMAGE), + recursive=True): + sample_id = osp.relpath(image_path, self._path) \ + .replace(CityscapesPath.ORIGINAL_IMAGE, '') + anns = [] + instances_path = osp.join(annotations_path, sample_id + '_' + + CityscapesPath.GT_FINE_DIR + CityscapesPath.INSTANCES_IMAGE) + if osp.isfile(instances_path): + instances_mask = load_image(instances_path, dtype=np.int32) + segm_ids = np.unique(instances_mask) + for segm_id in segm_ids: + if segm_id < 1000: + semanticId = segm_id + isCrowd = True + ann_id = segm_id + else: + semanticId = segm_id // 1000 + isCrowd = False + ann_id = segm_id % 1000 + anns.append(Mask( + image=self._lazy_extract_mask(instances_mask, segm_id), + label=semanticId, id=ann_id, + attributes = { 'is_crowd': isCrowd })) + items[sample_id] = DatasetItem(id=sample_id, subset=self._subset, + image=image_path, annotations=anns) + return items + + @staticmethod + def _lazy_extract_mask(mask, c): + return lambda: mask == c + + +class CityscapesImporter(Importer): + @classmethod + def find_sources(cls, path): + return cls._find_sources_recursive(path, '', 'cityscapes', + dirname=osp.join(CityscapesPath.IMGS_FINE_DIR, + CityscapesPath.ORIGINAL_IMAGE_DIR), + max_depth=1) + + +LabelmapType = Enum('LabelmapType', ['cityscapes', 'source']) + +class CityscapesConverter(Converter): + DEFAULT_IMAGE_EXT = '.png' + + @staticmethod + def _get_labelmap(s): + if osp.isfile(s): + return s + try: + return LabelmapType[s].name + except KeyError: + import argparse + raise argparse.ArgumentTypeError() + + @classmethod + def build_cmdline_parser(cls, **kwargs): + parser = super().build_cmdline_parser(**kwargs) + + parser.add_argument('--apply-colormap', type=str_to_bool, default=True, + help="Use colormap for class masks (default: %(default)s)") + parser.add_argument('--label-map', type=cls._get_labelmap, default=None, + help="Labelmap file path or one of %s" % \ + ', '.join(t.name for t in LabelmapType)) + return parser + + def __init__(self, extractor, save_dir, + apply_colormap=True, label_map=None, **kwargs): + super().__init__(extractor, save_dir, **kwargs) + + self._apply_colormap = apply_colormap + + if label_map is None: + label_map = LabelmapType.source.name + self._load_categories(label_map) + + def apply(self): + os.makedirs(self._save_dir, exist_ok=True) + + for subset_name, subset in self._extractor.subsets().items(): + for item in subset: + image_path = osp.join(CityscapesPath.IMGS_FINE_DIR, + CityscapesPath.ORIGINAL_IMAGE_DIR, subset_name, + item.id + CityscapesPath.ORIGINAL_IMAGE) + if self._save_images: + self._save_image(item, osp.join(self._save_dir, image_path)) + + common_folder_path = osp.join(CityscapesPath.GT_FINE_DIR, + subset_name) + + masks = [a for a in item.annotations + if a.type == AnnotationType.mask] + if not masks: + continue + + common_image_name = item.id + '_' + CityscapesPath.GT_FINE_DIR + + compiled_class_mask = CompiledMask.from_instance_masks(masks, + instance_labels=[self._label_id_mapping(m.label) + for m in masks]) + color_mask_path = osp.join(common_folder_path, + common_image_name + CityscapesPath.COLOR_IMAGE) + self.save_mask(osp.join(self._save_dir, color_mask_path), + compiled_class_mask.class_mask) + + labelids_mask_path = osp.join(common_folder_path, + common_image_name + CityscapesPath.LABELIDS_IMAGE) + self.save_mask(osp.join(self._save_dir, labelids_mask_path), + compiled_class_mask.class_mask, apply_colormap=False, + dtype=np.int32) + + compiled_instance_mask = CompiledMask.from_instance_masks(masks, + instance_labels=[m.id if m.attributes.get('is_crowd', True) + else m.label * 1000 + m.id for m in masks]) + inst_path = osp.join(common_folder_path, + common_image_name + CityscapesPath.INSTANCES_IMAGE) + self.save_mask(osp.join(self._save_dir, inst_path), + compiled_instance_mask.class_mask, apply_colormap=False, + dtype=np.int32) + self.save_label_map() + + def save_label_map(self): + path = osp.join(self._save_dir, CityscapesPath.LABELMAP_FILE) + write_label_map(path, self._label_map) + + def _load_categories(self, label_map_source): + if label_map_source == LabelmapType.cityscapes.name: + # use the default Cityscapes colormap + label_map = CityscapesLabelMap + + elif label_map_source == LabelmapType.source.name and \ + AnnotationType.mask not in self._extractor.categories(): + # generate colormap for input labels + labels = self._extractor.categories() \ + .get(AnnotationType.label, LabelCategories()) + label_map = OrderedDict((item.name, None) + for item in labels.items) + + elif label_map_source == LabelmapType.source.name and \ + AnnotationType.mask in self._extractor.categories(): + # use source colormap + labels = self._extractor.categories()[AnnotationType.label] + colors = self._extractor.categories()[AnnotationType.mask] + label_map = OrderedDict() + for idx, item in enumerate(labels.items): + color = colors.colormap.get(idx) + if color is not None: + label_map[item.name] = color + + elif isinstance(label_map_source, dict): + label_map = OrderedDict( + sorted(label_map_source.items(), key=lambda e: e[0])) + + elif isinstance(label_map_source, str) and osp.isfile(label_map_source): + label_map = parse_label_map(label_map_source) + + else: + raise Exception("Wrong labelmap specified, " + "expected one of %s or a file path" % \ + ', '.join(t.name for t in LabelmapType)) + + self._categories = make_cityscapes_categories(label_map) + self._label_map = label_map + self._label_id_mapping = self._make_label_id_map() + + def _make_label_id_map(self): + map_id, id_mapping, src_labels, dst_labels = make_label_id_mapping( + self._extractor.categories().get(AnnotationType.label), + self._categories[AnnotationType.label]) + + void_labels = [src_label for src_id, src_label in src_labels.items() + if src_label not in dst_labels] + if void_labels: + log.warning("The following labels are remapped to background: %s" % + ', '.join(void_labels)) + log.debug("Saving segmentations with the following label mapping: \n%s" % + '\n'.join(["#%s '%s' -> #%s '%s'" % + ( + src_id, src_label, id_mapping[src_id], + self._categories[AnnotationType.label] \ + .items[id_mapping[src_id]].name + ) + for src_id, src_label in src_labels.items() + ]) + ) + + return map_id + + def save_mask(self, path, mask, colormap=None, apply_colormap=True, + dtype=np.uint8): + if self._apply_colormap and apply_colormap: + if colormap is None: + colormap = self._categories[AnnotationType.mask].colormap + mask = paint_mask(mask, colormap) + save_image(path, mask, create_dir=True, dtype=dtype) diff --git a/datumaro/util/image.py b/datumaro/util/image.py index 17e2a0d0ba..e1acd4792d 100644 --- a/datumaro/util/image.py +++ b/datumaro/util/image.py @@ -65,7 +65,13 @@ def save_image(path, image, create_dir=False, dtype=np.uint8, **kwargs): if not kwargs: kwargs = {} - if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2: + # NOTE: OpenCV documentation says "If the image format is not supported, + # the image will be converted to 8-bit unsigned and saved that way". + # Conversion from np.int32 to np.uint8 is not working properly + backend = _IMAGE_BACKEND + if dtype == np.int32: + backend = _IMAGE_BACKENDS.PIL + if backend == _IMAGE_BACKENDS.cv2: import cv2 params = [] @@ -78,7 +84,7 @@ def save_image(path, image, create_dir=False, dtype=np.uint8, **kwargs): image = image.astype(dtype) cv2.imwrite(path, image, params=params) - elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL: + elif backend == _IMAGE_BACKENDS.PIL: from PIL import Image params = {} diff --git a/docs/design.md b/docs/design.md index 1e520400c0..b24a57a595 100644 --- a/docs/design.md +++ b/docs/design.md @@ -108,7 +108,7 @@ It should be capable of downloading and processing data from CVAT. - [x] PASCAL VOC - [x] YOLO - [x] TF Detection API - - [ ] Cityscapes + - [x] Cityscapes - [x] ImageNet - Dataset visualization (`show`) diff --git a/docs/formats/cityscapes_user_manual.md b/docs/formats/cityscapes_user_manual.md new file mode 100644 index 0000000000..f8e98b71b3 --- /dev/null +++ b/docs/formats/cityscapes_user_manual.md @@ -0,0 +1,176 @@ +# Cityscapes user manual + +## Contents + +- [Format specification](#format-specification) +- [Load Cityscapes dataset](#load-Cityscapes-dataset) +- [Export to other formats](#export-to-other-formats) +- [Export to Cityscapes](#export-to-Cityscapes) +- [Particular use cases](#particular-use-cases) + +## Format specification + +Cityscapes format overview available [here](https://www.cityscapes-dataset.com/dataset-overview/). +Cityscapes format specification available [here](https://github.com/mcordts/cityscapesScripts#the-cityscapes-dataset). + +Cityscapes dataset format supports `Masks` (segmentations tasks) annotations. + +## Load Cityscapes dataset + +The Cityscapes dataset is available for free [download](https://www.cityscapes-dataset.com/downloads/). + +There are two ways to create Datumaro project and add Cityscapes dataset to it: + +``` bash +datum import --format cityscapes --input-path +# or +datum create +datum add path -f cityscapes +``` + +It is possible to specify project name and project directory run +`datum create --help` for more information. + +Cityscapes dataset directory should have the following structure: + + +``` +└─ Dataset/ + ├── imgsFine/ + │ ├── leftImg8bit + │ │ ├── + │ │ | ├── {city1} + │ │ │ | ├── {city1}_{seq:[0...6]}_{frame:[0...6]}_leftImg8bit.png + │ │ │ │ └── ... + │ │ | ├── {city2} + │ │ │ └── ... + │ │ └── ... + ├── gtFine/ + │ ├── + │ │ ├── {city1} + │ │ | ├── {city1}_{seq:[0...6]}_{frame:[0...6]}_gtFine_color.png + │ │ | ├── {city1}_{seq:[0...6]}_{frame:[0...6]}_gtFine_instanceIds.png + │ │ | ├── {city1}_{seq:[0...6]}_{frame:[0...6]}_gtFine_labelIds.png + │ │ │ └── ... + │ │ ├── {city2} + │ │ └── ... + │ └── ... +``` + +Annotated files description: +1. *leftImg8bit.png - left images in 8-bit LDR format +1. *color.png - class labels are encoded by its color +1. *instanceIds.png - class and instance labels are encoded by an instance ID. + The pixel values encode class and the individual instance: the integer part + of a division by 1000 of each ID provides class ID, the remainder + is the instance ID. If a certain annotation describes multiple instances, + then the pixels have the regular ID of that class +1. *labelIds.png - class labels are encoded by its ID + +To make sure that the selected dataset has been added to the project, you can run +`datum info`, which will display the project and dataset information. + +## Export to other formats + +Datumaro can convert Cityscapes dataset into any other format [Datumaro supports](../user_manual.md#supported-formats). +To get the expected result, the dataset needs to be converted to formats +that support the segmentation task (e.g. PascalVOC, CamVID, etc.) +There are few ways to convert Cityscapes dataset to other dataset format: + +``` bash +datum project import -f cityscapes -i +datum export -f voc -o +# or +datum convert -if cityscapes -i -f voc -o +``` + +Some formats provide extra options for conversion. +These options are passed after double dash (`--`) in the command line. +To get information about them, run + +`datum export -f -- -h` + +## Export to Cityscapes + +There are few ways to convert dataset to Cityscapes format: + +``` bash +# export dataset into Cityscapes format from existing project +datum export -p -f cityscapes -o \ + -- --save-images +# converting to Cityscapes format from other format +datum convert -if voc -i \ + -f cityscapes -o -- --save-images +``` + +Extra options for export to cityscapes format: +- `--save-images` allow to export dataset with saving images +(by default `False`); +- `--image-ext IMAGE_EXT` allow to specify image extension +for exporting dataset (by default - keep original or use `.png`, if none). +- `--apply-colormap APPLY_COLORMAP` allow to use colormap for class masks +(`*color.png` files, by default `True`); +- `--label_map` allow to define a custom colormap. Example + +``` bash +# mycolormap.txt : +# 0 0 255 sky +# 255 0 0 person +#... +datum export -f cityscapes -- --label-map mycolormap.txt + +# or you can use original cityscapes colomap: +datum export -f cityscapes -- --label-map cityscapes +``` + +## Particular use cases + +Datumaro supports filtering, transformation, merging etc. for all formats +and for the Cityscapes format in particular. Follow +[user manual](../user_manual.md) +to get more information about these operations. + +There are few examples of using Datumaro operations to solve +particular problems with Cityscapes dataset: + +### Example 1. How to load an original Cityscapes dataset ans convert to Pascal VOC + +```bash +datum create -o project +datum add path -p project -f cityscapes ./Cityscapes/ +datum stats -p project +datum export -p final_project -o dataset -f voc --overwrite -- --save-images +``` + +### Example 2. How to create custom Cityscapes-like dataset + +```python +import numpy as np +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import Mask, DatasetItem + +import datumaro.plugins.cityscapes_format as Cityscapes + +label_map = OrderedDict() +label_map['background'] = (0, 0, 0) +label_map['label_1'] = (1, 2, 3) +label_map['label_2'] = (3, 2, 1) +categories = Cityscapes.make_cityscapes_categories(label_map) + +dataset = Dataset.from_iterable([ + DatasetItem(id=1, + image=np.ones((1, 5, 3)), + annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=1, id=1, + attributes={'is_crowd': False}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=2, id=2, + attributes={'is_crowd': False}), + ] + ), + ], categories=categories) + +dataset.export('./dataset', format='cityscapes') +``` + +More examples of working with Cityscapes dataset from code can be found in +[tests](../../tests/test_cityscapes_format.py) diff --git a/docs/user_manual.md b/docs/user_manual.md index 5e5a5e22c0..b757c4b1ad 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -131,6 +131,10 @@ List of supported formats: - CamVid (`segmentation`) - [Format specification](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) - [Dataset example](../tests/assets/camvid_dataset) +- Cityscapes (`segmentation`) + - [Format specification](https://www.cityscapes-dataset.com/dataset-overview/) + - [Dataset example](../tests/assets/cityscapes_dataset) + - [Format documentation](./formats/cityscapes_user_manual.md) - CVAT - [Format specification](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) - [Dataset example](../tests/assets/cvat_dataset) diff --git a/tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000031_gtFine_instanceIds.png b/tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000031_gtFine_instanceIds.png new file mode 100644 index 0000000000000000000000000000000000000000..9a2cb23ffa86206a2ab160a7130f677f754e7ec7 GIT binary patch literal 76 zcmeAS@N?(olHy`uVBq!ia0vp^tU$~t03;ZaS)bFS$tekG2`6-t6H*cw Y7+8KVJQc0h?*u9HboFyt=akR{04LoMdjJ3c literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000032_gtFine_instanceIds.png b/tests/assets/cityscapes_dataset/gtFine/test/defaultcity/defaultcity_000001_000032_gtFine_instanceIds.png new file mode 100644 index 0000000000000000000000000000000000000000..56c008eac13067d18945da44505d2462602153bc GIT binary patch literal 76 zcmeAS@N?(olHy`uVBq!ia0vp^tU$~t03;ZaS)bFS$!n&hBxL+Q^I>iy YBZELaGi#|D%K?xwPgg&ebxsLQ08Wk)i2wiq literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/gtFine/train/defaultcity/defaultcity_000002_000045_gtFine_instanceIds.png b/tests/assets/cityscapes_dataset/gtFine/train/defaultcity/defaultcity_000002_000045_gtFine_instanceIds.png new file mode 100644 index 0000000000000000000000000000000000000000..e658ec33cda661d28014cdf6891a175b988e7cff GIT binary patch literal 76 zcmeAS@N?(olHy`uVBq!ia0vp^tU$~t03;ZaS)bFS$q5N*34OLF4xBl_ Y!@y?6sC^-1(E^Y%Pgg&ebxsLQ06=^akN^Mx literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/gtFine/val/defaultcity/defaultcity_000001_000019_gtFine_instanceIds.png b/tests/assets/cityscapes_dataset/gtFine/val/defaultcity/defaultcity_000001_000019_gtFine_instanceIds.png new file mode 100644 index 0000000000000000000000000000000000000000..d2ccdd1f1a7ced01bd18144787ccba00b5db98a2 GIT binary patch literal 76 zcmeAS@N?(olHy`uVBq!ia0vp^tU$~t03;ZaS)bFS$q5N*i3y27GJ%bO WA&WsMXzu$7AZ4DeelF{r5}E)R;}Hu0 literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000031_leftImg8bit.png b/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000031_leftImg8bit.png new file mode 100644 index 0000000000000000000000000000000000000000..528f10546704be6b339cfe1f577ca4b10ef4f472 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1j!2~2{&iT9qEaBo9!XcZ?!o;QmFVdQ&MBb@0GX=|x&QzG literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000032_leftImg8bit.png b/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/test/defaultcity/defaultcity_000001_000032_leftImg8bit.png new file mode 100644 index 0000000000000000000000000000000000000000..528f10546704be6b339cfe1f577ca4b10ef4f472 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1j!2~2{&iT9qEaBo9!XcZ?!o;QmFVdQ&MBb@0GX=|x&QzG literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/train/defaultcity/defaultcity_000002_000045_leftImg8bit.png b/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/train/defaultcity/defaultcity_000002_000045_leftImg8bit.png new file mode 100644 index 0000000000000000000000000000000000000000..528f10546704be6b339cfe1f577ca4b10ef4f472 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1j!2~2{&iT9qEaBo9!XcZ?!o;QmFVdQ&MBb@0GX=|x&QzG literal 0 HcmV?d00001 diff --git a/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/val/defaultcity/defaultcity_000001_000019_leftImg8bit.png b/tests/assets/cityscapes_dataset/imgsFine/leftImg8bit/val/defaultcity/defaultcity_000001_000019_leftImg8bit.png new file mode 100644 index 0000000000000000000000000000000000000000..528f10546704be6b339cfe1f577ca4b10ef4f472 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1j!2~2{&iT9qEaBo9!XcZ?!o;QmFVdQ&MBb@0GX=|x&QzG literal 0 HcmV?d00001 diff --git a/tests/test_cityscapes_format.py b/tests/test_cityscapes_format.py new file mode 100644 index 0000000000..fd23de9d76 --- /dev/null +++ b/tests/test_cityscapes_format.py @@ -0,0 +1,350 @@ +import os.path as osp +from collections import OrderedDict +from functools import partial +from unittest import TestCase + +import datumaro.plugins.cityscapes_format as Cityscapes +import numpy as np +from datumaro.components.extractor import (AnnotationType, DatasetItem, + Extractor, LabelCategories, Mask) +from datumaro.components.dataset import Dataset +from datumaro.plugins.cityscapes_format import (CityscapesImporter, + CityscapesConverter) +from datumaro.util.image import Image +from datumaro.util.test_utils import (TestDir, compare_datasets, + test_save_and_load) + +DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', + 'cityscapes_dataset') + +class CityscapesFormatTest(TestCase): + def test_can_write_and_parse_labelmap(self): + src_label_map = Cityscapes.CityscapesLabelMap + + with TestDir() as test_dir: + file_path = osp.join(test_dir, 'label_colors.txt') + + Cityscapes.write_label_map(file_path, src_label_map) + dst_label_map = Cityscapes.parse_label_map(file_path) + + self.assertEqual(src_label_map, dst_label_map) + +class CityscapesImportTest(TestCase): + def test_can_import(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='defaultcity/defaultcity_000001_000031', + subset='test', + image=np.ones((1, 5, 3)), + annotations=[ + Mask(image=np.array([[1, 1, 0, 0, 0]]), id=3, label=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 0, 1, 0, 0]]), id=1, label=27, + attributes={'is_crowd': False}), + Mask(image=np.array([[0, 0, 0, 1, 1]]), id=2, label=27, + attributes={'is_crowd': False}), + ] + ), + DatasetItem(id='defaultcity/defaultcity_000001_000032', + subset='test', + image=np.ones((1, 5, 3)), + annotations=[ + Mask(image=np.array([[1, 1, 0, 0, 0]]), id=1, label=31, + attributes={'is_crowd': False}), + Mask(image=np.array([[0, 0, 1, 0, 0]]), id=12, label=12, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 0, 0, 1, 1]]), id=3, label=3, + attributes={'is_crowd': True}), + ] + ), + DatasetItem(id='defaultcity/defaultcity_000002_000045', + subset='train', + image=np.ones((1, 5, 3)), + annotations=[ + Mask(image=np.array([[1, 1, 0, 1, 1]]), id=3, label=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 0, 1, 0, 0]]), id=1, label=24, + attributes={'is_crowd': False}), + ] + ), + DatasetItem(id='defaultcity/defaultcity_000001_000019', + subset = 'val', + image=np.ones((1, 5, 3)), + annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), id=3, label=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), id=24, label=1, + attributes={'is_crowd': False}), + ] + ), + ], categories=Cityscapes.make_cityscapes_categories()) + + parsed_dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'cityscapes') + + compare_datasets(self, source_dataset, parsed_dataset) + + def test_can_detect_cityscapes(self): + self.assertTrue(CityscapesImporter.detect(DUMMY_DATASET_DIR)) + + +class TestExtractorBase(Extractor): + def _label(self, cityscapes_label): + return self.categories()[AnnotationType.label].find(cityscapes_label)[0] + + def categories(self): + return Cityscapes.make_cityscapes_categories() + +class CityscapesConverterTest(TestCase): + def _test_save_and_load(self, source_dataset, converter, test_dir, + target_dataset=None, importer_args=None, **kwargs): + return test_save_and_load(self, source_dataset, converter, test_dir, + importer='cityscapes', + target_dataset=target_dataset, importer_args=importer_args, **kwargs) + + def test_can_save_cityscapes_segm(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='defaultcity_1_2', subset='test', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[0, 0, 0, 1, 0]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=24, id=1, + attributes={'is_crowd': False}), + Mask(image=np.array([[1, 0, 0, 0, 1]]), label=15, id=15, + attributes={'is_crowd': True}), + ]), + DatasetItem(id='defaultcity_3', subset='val', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 1, 0, 1, 1]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 0, 1, 0, 0]]), label=5, id=5, + attributes={'is_crowd': True}), + ]), + ]) + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_can_save_cityscapes_segm_unpainted(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='defaultcity_1_2', subset='test', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[0, 0, 0, 1, 0]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=24, id=1, + attributes={'is_crowd': False}), + Mask(image=np.array([[1, 0, 0, 0, 1]]), label=15, id=15, + attributes={'is_crowd': True}), + ]), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True, apply_colormap=False), test_dir) + + def test_can_save_cityscapes_dataset_with_no_subsets(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='defaultcity_1_2', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 0]]), label=0, id=0, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 1]]), label=3, id=3, + attributes={'is_crowd': True}), + ]), + + DatasetItem(id='defaultcity_1_3', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 1, 0, 1, 0]]), label=1, id=1, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 0, 1, 0, 1]]), label=2, id=2, + attributes={'is_crowd': True}), + ]), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_can_save_cityscapes_dataset_without_frame_and_sequence(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='justcity', subset='test', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=24, id=1, + attributes={'is_crowd': False}), + ]), + ]) + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=24, id=1, + attributes={'is_crowd': False}), + ]), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_can_save_cityscapes_dataset_with_strange_id(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='a/b/1', subset='test', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=3, id=3, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=24, id=1, + attributes={'is_crowd': False}), + ]), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_can_save_with_no_masks(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='city_1_2', subset='test', + image=np.ones((2, 5, 3)), + ), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, label_map='cityscapes', + save_images=True), test_dir) + + def test_dataset_with_source_labelmap_undefined(self): + class SrcExtractor(TestExtractorBase): + def __iter__(self): + yield DatasetItem(id=1, image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=1, id=1, + attributes={'is_crowd': False}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=2, id=2, + attributes={'is_crowd': False}), + ]) + + def categories(self): + label_cat = LabelCategories() + label_cat.add('background') + label_cat.add('Label_1') + label_cat.add('label_2') + return { + AnnotationType.label: label_cat, + } + + class DstExtractor(TestExtractorBase): + def __iter__(self): + yield DatasetItem(id=1, image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), + attributes={'is_crowd': False}, id=1, + label=self._label('Label_1')), + Mask(image=np.array([[0, 1, 1, 0, 0]]), + attributes={'is_crowd': False}, id=2, + label=self._label('label_2')), + ]) + + def categories(self): + label_map = OrderedDict() + label_map['background'] = None + label_map['Label_1'] = None + label_map['label_2'] = None + return Cityscapes.make_cityscapes_categories(label_map) + + with TestDir() as test_dir: + self._test_save_and_load(SrcExtractor(), + partial(CityscapesConverter.convert, label_map='source', + save_images=True), test_dir, target_dataset=DstExtractor()) + + def test_dataset_with_source_labelmap_defined(self): + class SrcExtractor(TestExtractorBase): + def __iter__(self): + yield DatasetItem(id=1, image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), label=1, id=1, + attributes={'is_crowd': False}), + Mask(image=np.array([[0, 1, 1, 0, 0]]), label=2, id=2, + attributes={'is_crowd': False}), + ]) + + def categories(self): + label_map = OrderedDict() + label_map['background'] = (0, 0, 0) + label_map['label_1'] = (1, 2, 3) + label_map['label_2'] = (3, 2, 1) + return Cityscapes.make_cityscapes_categories(label_map) + + class DstExtractor(TestExtractorBase): + def __iter__(self): + yield DatasetItem(id=1, image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 1]]), + attributes={'is_crowd': False}, id=1, + label=self._label('label_1')), + Mask(image=np.array([[0, 1, 1, 0, 0]]), + attributes={'is_crowd': False}, id=2, + label=self._label('label_2')), + ]) + + def categories(self): + label_map = OrderedDict() + label_map['background'] = (0, 0, 0) + label_map['label_1'] = (1, 2, 3) + label_map['label_2'] = (3, 2, 1) + return Cityscapes.make_cityscapes_categories(label_map) + + with TestDir() as test_dir: + self._test_save_and_load(SrcExtractor(), + partial(CityscapesConverter.convert, label_map='source', + save_images=True), test_dir, target_dataset=DstExtractor()) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + + DatasetItem(id='a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.ones((1, 5, 3)) + ), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 0]]), label=0, id=0, + attributes={'is_crowd': True}), + Mask(image=np.array([[0, 1, 1, 0, 1]]), label=1, id=1, + attributes={'is_crowd': True}), + ]), + ]) + + def categories(self): + label_map = OrderedDict() + label_map['a'] = None + label_map['b'] = None + return Cityscapes.make_cityscapes_categories(label_map) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CityscapesConverter.convert, save_images=True), + test_dir, require_images=True) From bac10c760ea417f02e8c84d2a8d38d6f249cb24d Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Wed, 26 May 2021 14:42:58 +0300 Subject: [PATCH 26/40] Fix saving attribute in WiderFace extractor (#251) * add fixes * update changelog --- CHANGELOG.md | 1 + datumaro/plugins/widerface_format.py | 6 +++++- tests/test_widerface_format.py | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e85315d9d1..2be78b9415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allowed adding "difficult", "truncated", "occluded" attributes when converting to Pascal VOC if these attributes are not present () - Empty lines in YOLO annotations are ignored () - Export in VOC format when no image info is available () +- Fixed saving attribute in WiderFace extractor () ### Security - diff --git a/datumaro/plugins/widerface_format.py b/datumaro/plugins/widerface_format.py index 96796b09f3..a8439dc83c 100644 --- a/datumaro/plugins/widerface_format.py +++ b/datumaro/plugins/widerface_format.py @@ -10,6 +10,7 @@ from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, SourceExtractor) +from datumaro.util import str_to_bool class WiderFacePath: @@ -119,7 +120,10 @@ def _load_items(self, path): i = 4 for attr in WiderFacePath.BBOX_ATTRIBUTES: if bbox_list[i] != '-': - attributes[attr] = bbox_list[i] + if bbox_list[i] in ['True', 'False']: + attributes[attr] = str_to_bool(bbox_list[i]) + else: + attributes[attr] = bbox_list[i] i += 1 annotations.append(Bbox( diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index 8e2586999e..4d733ee82a 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -43,7 +43,9 @@ def test_can_save_and_load(self): 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), Bbox(0, 2, 3, 2, label=0, attributes={ - 'occluded': 'False'}), + 'occluded': False}), + Bbox(0, 3, 4, 2, label=0, attributes={ + 'occluded': True}), Bbox(0, 2, 4, 2, label=0), Bbox(0, 7, 3, 2, label=0, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', From 8fd0831831bf2fcb571c74f49114deb888f345cb Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Wed, 26 May 2021 15:12:47 +0300 Subject: [PATCH 27/40] Fix spelling errors (#252) --- CHANGELOG.md | 4 ++-- README.md | 4 ++-- datumaro/components/extractor.py | 6 +++--- datumaro/components/operations.py | 6 +++--- datumaro/plugins/accuracy_checker_plugin/details/ac.py | 4 ++-- .../accuracy_checker_plugin/details/representation.py | 4 ++-- datumaro/plugins/cifar_format.py | 4 ++-- datumaro/plugins/ndr.py | 6 +++--- datumaro/plugins/splitter.py | 6 +++--- docs/cli_design.mm | 2 +- docs/developer_guide.md | 2 +- docs/formats/cityscapes_user_manual.md | 2 +- docs/formats/coco_user_manual.md | 2 +- docs/user_manual.md | 2 +- tests/cli/test_voc_format.py | 6 +++--- tests/test_dataset.py | 2 +- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be78b9415..d05fc35f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Support for escaping in attribiute values in LabelMe format () +- Support for escaping in attribute values in LabelMe format () - Support for Segmentation Splitting () - Support for CIFAR-10/100 dataset format (, ) - Support for COCO panoptic and stuff format () @@ -176,7 +176,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `CamVid` dataset format () - Ability to install `opencv-python-headless` dependency with `DATUMARO_HEADLESS=1` - enviroment variable instead of `opencv-python` () + environment variable instead of `opencv-python` () ### Changed - Allow empty supercategory in COCO () diff --git a/README.md b/README.md index 2c5957f522..559eac4762 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ CVAT annotations ---> Publication, statistics etc. - keep only vertically-oriented images - remove small area bounding boxes from annotations - Annotation conversions, for instance: - - polygons to instance masks and vise-versa + - polygons to instance masks and vice-versa - apply a custom colormap for mask annotations - rename or remove dataset labels - Splitting a dataset into multiple subsets like `train`, `val`, and `test`: @@ -170,7 +170,7 @@ CVAT annotations ---> Publication, statistics etc. - sampling with Entropy based algorithm - Dataset quality checking - Simple checking for errors - - Comparison with model infernece + - Comparison with model inference - Merging and comparison of multiple datasets - Annotation validation based on the task type(classification, etc) - Dataset comparison diff --git a/datumaro/components/extractor.py b/datumaro/components/extractor.py index ebeaf01ecc..e8cc4f89c9 100644 --- a/datumaro/components/extractor.py +++ b/datumaro/components/extractor.py @@ -1,5 +1,5 @@ -# Copyright (C) 2019-2020 Intel Corporation +# Copyright (C) 2019-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -69,7 +69,7 @@ def from_iterable(cls, iterable): iterable ([type]): This iterable object can be: 1)simple str - will generate one Category with str as name 2)list of str - will interpreted as list of Category names - 3)list of positional argumetns - will generate Categories + 3)list of positional arguments - will generate Categories with this arguments @@ -448,7 +448,7 @@ def from_iterable(cls, iterable): Args: iterable ([type]): This iterable object can be: - 1) list of positional argumetns - will generate Categories + 1) list of positional arguments - will generate Categories with these arguments Returns: diff --git a/datumaro/components/operations.py b/datumaro/components/operations.py index db3c9c3889..bfe26aec99 100644 --- a/datumaro/components/operations.py +++ b/datumaro/components/operations.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -153,7 +153,7 @@ class Conf: quorum = attrib(converter=int, default=0) ignored_attributes = attrib(converter=set, factory=set) - def _groups_conveter(value): + def _groups_converter(value): result = [] for group in value: rg = set() @@ -163,7 +163,7 @@ def _groups_conveter(value): rg.add((name, optional)) result.append(rg) return result - groups = attrib(converter=_groups_conveter, factory=list) + groups = attrib(converter=_groups_converter, factory=list) close_distance = attrib(converter=float, default=0.75) conf = attrib(converter=ensure_cls(Conf), factory=Conf) diff --git a/datumaro/plugins/accuracy_checker_plugin/details/ac.py b/datumaro/plugins/accuracy_checker_plugin/details/ac.py index 4fc2ffb5c6..b235e57848 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/ac.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/ac.py @@ -1,10 +1,10 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT from datumaro.util.tf_util import import_tf -import_tf() # prevent TF loading and potential interpeter crash +import_tf() # prevent TF loading and potential interpreter crash from itertools import groupby diff --git a/datumaro/plugins/accuracy_checker_plugin/details/representation.py b/datumaro/plugins/accuracy_checker_plugin/details/representation.py index d7007806bf..76da49ee0a 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/representation.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/representation.py @@ -1,10 +1,10 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT from datumaro.util.tf_util import import_tf -import_tf() # prevent TF loading and potential interpeter crash +import_tf() # prevent TF loading and potential interpreter crash import accuracy_checker.representation as ac diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index 287f00de8e..e2a2bec12b 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -161,7 +161,7 @@ def apply(self): annotation_dict['data'] = np.array(data) if len(image_sizes): size = (CifarPath.IMAGE_SIZE, CifarPath.IMAGE_SIZE) - # 'image_sizes' isn't included in the standart format, + # 'image_sizes' isn't included in the standard format, # needed for different image sizes annotation_dict['image_sizes'] = [image_sizes.get(p, size) for p in range(len(data))] diff --git a/datumaro/plugins/ndr.py b/datumaro/plugins/ndr.py index df82e17935..b92945bbad 100644 --- a/datumaro/plugins/ndr.py +++ b/datumaro/plugins/ndr.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -173,7 +173,7 @@ def _remove(self): (item.id, len(img.shape))) if self.algorithm == Algorithm.gradient: - # Caculate gradient + # Calculate gradient img = self._cgrad_feature(img) else : raise NotImplementedError() @@ -267,7 +267,7 @@ def _keep_cut(self, num_cut, all_key, fidx, # of the collision(number of same hash key) # [x1, x2, y1, y2, y3, y4, z1, z2, z3]. x, y and z for hash key # i.e. there are 4 elements which have hash key y. - # then the occurence will be [2, 4, 3] and reverse of them + # then the occurrence will be [2, 4, 3] and reverse of them # will be [1/2, 1/4, 1/3] # Normalizing them by dividing with sum, we get [6/13, 3/13, 4/13] # Then the key x will be sampled with probability 6/13 diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index 786828b3c5..5d414e1333 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -31,7 +31,7 @@ class Split(Transform, CliPlugin): Each image can have multiple object annotations - (bbox, mask, polygon). Since an image shouldn't be included in multiple subsets at the same time, and image annotations - shoudln't be split, in general, dataset annotations are unlikely + shouldn't be split, in general, dataset annotations are unlikely to be split exactly in the specified ratio. |n This split tries to split dataset images as close as possible to the specified ratio, keeping the initial class distribution.|n @@ -684,7 +684,7 @@ class _InstanceSpecificSplit(_TaskSpecificSplit): each image can have multiple object annotations - (instance bounding boxes, masks, polygons). Since an image shouldn't be included in multiple subsets at the same time, and image annotations - shoudln't be split, in general, dataset annotations are unlikely to be split + shouldn't be split, in general, dataset annotations are unlikely to be split exactly in the specified ratio. |n This split tries to split dataset images as close as possible to the specified ratio, keeping the initial class distribution.|n diff --git a/docs/cli_design.mm b/docs/cli_design.mm index 0ff17cb299..9e2eddaceb 100644 --- a/docs/cli_design.mm +++ b/docs/cli_design.mm @@ -9,7 +9,7 @@ - + diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 2bfab0e5b8..e8eff1bcf8 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -72,7 +72,7 @@ class colors for masks, class attributes. This information is stored in `dataset.categories`, which is a mapping from `AnnotationType` to a corresponding `...Categories` class. Each annotation type can have its `Categories`. Typically, there will be a `LabelCategories` object. -Annotations and other categories adress dataset labels +Annotations and other categories address dataset labels by their indices in this object. The main operation for a dataset is iteration over its elements. diff --git a/docs/formats/cityscapes_user_manual.md b/docs/formats/cityscapes_user_manual.md index f8e98b71b3..534f91c726 100644 --- a/docs/formats/cityscapes_user_manual.md +++ b/docs/formats/cityscapes_user_manual.md @@ -133,7 +133,7 @@ to get more information about these operations. There are few examples of using Datumaro operations to solve particular problems with Cityscapes dataset: -### Example 1. How to load an original Cityscapes dataset ans convert to Pascal VOC +### Example 1. How to load an original Cityscapes dataset and convert to Pascal VOC ```bash datum create -o project diff --git a/docs/formats/coco_user_manual.md b/docs/formats/coco_user_manual.md index 2f824dd11a..5b2d379016 100644 --- a/docs/formats/coco_user_manual.md +++ b/docs/formats/coco_user_manual.md @@ -181,7 +181,7 @@ to get more information about these operations. There are few examples of using Datumaro operations to solve particular problems with COCO dataset: -### Example 1. How to load an original panoptic COCO dataset ans convert to Pascal VOC +### Example 1. How to load an original panoptic COCO dataset and convert to Pascal VOC ```bash datum create -o project diff --git a/docs/user_manual.md b/docs/user_manual.md index b757c4b1ad..2a83cec341 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1348,7 +1348,7 @@ pip install 'git+https://github.com/openvinotoolkit/open_model_zoo.git#subdirect #### OpenVINO™ This plugin provides support for model inference with [OpenVINO™](https://01.org/openvinotoolkit). -The plugin depends on the OpenVINO™ Tookit, which can be installed by +The plugin depends on the OpenVINO™ Toolkit, which can be installed by following [these instructions](https://docs.openvinotoolkit.org/latest/index.html#packaging_and_deployment) ### Dataset Formats diff --git a/tests/cli/test_voc_format.py b/tests/cli/test_voc_format.py index c1516bcda2..8a686e4f58 100644 --- a/tests/cli/test_voc_format.py +++ b/tests/cli/test_voc_format.py @@ -68,13 +68,13 @@ def test_preparing_dataset_for_train_model(self): run(self, 'filter', '-p', test_dir, '-m', 'i+a', '-e', "/item/annotation[occluded='False']", '-o', result_path) - splitted_path = osp.join(test_dir, 'splitted') - run(self, 'transform', '-p', result_path, '-o', splitted_path, + split_path = osp.join(test_dir, 'split') + run(self, 'transform', '-p', result_path, '-o', split_path, '-t', 'random_split', '--', '-s', 'test:.5', '-s', 'train:.5', '--seed', '1') export_path = osp.join(test_dir, 'dataset') - run(self, 'export', '-p', splitted_path, '-f', 'voc', + run(self, 'export', '-p', split_path, '-f', 'voc', '-o', export_path, '--', '--label-map', 'voc') parsed_dataset = Dataset.import_from(export_path, format='voc') diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 37f83c6f45..cb8a776c02 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -458,7 +458,7 @@ def __iter__(self): self.assertTrue(iter_called) - def test_can_chain_lazy_tranforms(self): + def test_can_chain_lazy_transforms(self): iter_called = False class TestExtractor(Extractor): def __iter__(self): From d42ff1498ff7276767f8f8a86d92d87535122920 Mon Sep 17 00:00:00 2001 From: Seungyoon Woo Date: Thu, 27 May 2021 19:59:44 +0900 Subject: [PATCH 28/40] Configurable Threshold CLI support (#250) * add validator cli * add configurable validator threshold * update changelog --- CHANGELOG.md | 1 + datumaro/cli/contexts/project/__init__.py | 8 +- datumaro/components/validator.py | 117 +++++++++++++++++----- tests/test_validator.py | 88 +++++++++------- 4 files changed, 155 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d05fc35f20..12f1f14d4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Documentation file for COCO format () - Documentation file and integration tests for YOLO format () - Support for Cityscapes dataset format () +- Support for Validator configurable threshold() ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/datumaro/cli/contexts/project/__init__.py b/datumaro/cli/contexts/project/__init__.py index 963f52158e..ff4dfb10bd 100644 --- a/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/cli/contexts/project/__init__.py @@ -808,6 +808,8 @@ def build_validate_parser(parser_ctor=argparse.ArgumentParser): help="Subset to validate (default: None)") parser.add_argument('-p', '--project', dest='project_dir', default='.', help="Directory of the project to validate (default: current dir)") + parser.add_argument('extra_args', nargs=argparse.REMAINDER, default=None, + help="Optional arguments for validator (pass '-- -h' for help)") parser.set_defaults(command=validate_command) return parser @@ -822,7 +824,11 @@ def validate_command(args): if subset_name is not None: dataset = dataset.get_subset(subset_name) dst_file_name += f'-{subset_name}' - validation_results = validate_annotations(dataset, task_type) + + extra_args = {} + from datumaro.components.validator import _Validator + extra_args = _Validator.parse_cmdline(args.extra_args) + validation_results = validate_annotations(dataset, task_type, **extra_args) def numpy_encoder(obj): if isinstance(obj, np.generic): diff --git a/datumaro/components/validator.py b/datumaro/components/validator.py index 2d03363475..1e910029f8 100644 --- a/datumaro/components/validator.py +++ b/datumaro/components/validator.py @@ -18,6 +18,7 @@ NegativeLength, InvalidValue, FarFromLabelMean, FarFromAttrMean, OnlyOneAttributeValue) from datumaro.components.extractor import AnnotationType, LabelCategories +from datumaro.components.cli_plugin import CliPlugin from datumaro.util import parse_str_enum_value @@ -26,13 +27,7 @@ TaskType = Enum('TaskType', ['classification', 'detection', 'segmentation']) -class _Validator: - DEFAULT_FEW_SAMPLES = 1 - DEFAULT_IMBALANCE_RATIO = 50 - DEFAULT_FAR_FROM_MEAN = 5.0 - DEFAULT_DOMINANCE_RATIO = 0.8 - DEFAULT_DOMINANCE_TOPK_BINS = 0.1 - +class _Validator(CliPlugin): # statistics templates numerical_stat_template = { 'items_far_from_mean': {}, @@ -64,7 +59,50 @@ class _Validator: Abstract method that must be implemented in a subclass. """ - def __init__(self, task_type=None): + @classmethod + def build_cmdline_parser(cls, **kwargs): + parser = super().build_cmdline_parser(**kwargs) + parser.add_argument('-fs', '--few_samples_thr', default=1, type=int, + help="Threshold for giving a warning for minimum number of" + "samples per class") + parser.add_argument('-ir', '--imbalance_ratio_thr', default=50, type=int, + help="Threshold for giving data imbalance warning;" + "IR(imbalance ratio) = majority/minority") + parser.add_argument('-m', '--far_from_mean_thr', default=5.0, type=float, + help="Threshold for giving a warning that data is far from mean;" + "A constant used to define mean +/- k * standard deviation;") + parser.add_argument('-dr', '--dominance_ratio_thr', default=0.8, type=float, + help="Threshold for giving a warning for bounding box imbalance;" + "Dominace_ratio = ratio of Top-k bin to total in histogram;") + parser.add_argument('-k', '--topk_bins', default=0.1, type=float, + help="Ratio of bins with the highest number of data" + "to total bins in the histogram; [0, 1]; 0.1 = 10%;") + return parser + + def __init__(self, task_type, few_samples_thr=None, + imbalance_ratio_thr=None, far_from_mean_thr=None, + dominance_ratio_thr=None, topk_bins=None): + """ + Validator + + Parameters + --------------- + few_samples_thr: int + minimum number of samples per class + warn user when samples per class is less than threshold + imbalance_ratio_thr: int + ratio of majority attribute to minority attribute + warn user when annotations are unevenly distributed + far_from_mean_thr: float + constant used to define mean +/- m * stddev + warn user when there are too big or small values + dominance_ratio_thr: float + ratio of Top-k bin to total + warn user when dominance ratio is over threshold + topk_bins: float + ratio of selected bins with most item number to total bins + warn user when values are not evenly distributed + """ self.task_type = parse_str_enum_value(task_type, TaskType, default=TaskType.classification) @@ -78,11 +116,11 @@ def __init__(self, task_type=None): self.ann_types = {AnnotationType.mask, AnnotationType.polygon} self.str_ann_type = "mask or polygon" - self.far_from_mean_thr = self.DEFAULT_FAR_FROM_MEAN - self.imbalance_ratio_thr = self.DEFAULT_IMBALANCE_RATIO - self.few_samples_thr = self.DEFAULT_FEW_SAMPLES - self.dominance_thr = self.DEFAULT_DOMINANCE_RATIO - self.topk_bins_ratio = self.DEFAULT_DOMINANCE_TOPK_BINS + self.few_samples_thr = few_samples_thr + self.imbalance_ratio_thr = imbalance_ratio_thr + self.far_from_mean_thr = far_from_mean_thr + self.dominance_thr = dominance_ratio_thr + self.topk_bins_ratio = topk_bins def _compute_common_statistics(self, dataset): defined_attr_template = { @@ -537,8 +575,13 @@ class ClassificationValidator(_Validator): A validator class for classification tasks. """ - def __init__(self): - super().__init__(TaskType.classification) + def __init__(self, few_samples_thr, imbalance_ratio_thr, + far_from_mean_thr, dominance_ratio_thr, topk_bins): + super().__init__(task_type=TaskType.classification, + few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, topk_bins=topk_bins) def _check_multi_label_annotations(self, stats): validation_reports = [] @@ -636,8 +679,13 @@ class DetectionValidator(_Validator): """ A validator class for detection tasks. """ - def __init__(self): - super().__init__(TaskType.detection) + def __init__(self, few_samples_thr, imbalance_ratio_thr, + far_from_mean_thr, dominance_ratio_thr, topk_bins): + super().__init__(task_type=TaskType.detection, + few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, topk_bins=topk_bins) def _check_negative_length(self, stats): validation_reports = [] @@ -917,8 +965,13 @@ class SegmentationValidator(_Validator): A validator class for (instance) segmentation tasks. """ - def __init__(self): - super().__init__(TaskType.segmentation) + def __init__(self, few_samples_thr, imbalance_ratio_thr, + far_from_mean_thr, dominance_ratio_thr, topk_bins): + super().__init__(task_type=TaskType.segmentation, + few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, topk_bins=topk_bins) def compute_statistics(self, dataset): """ @@ -1149,7 +1202,7 @@ def generate_reports(self, stats): return reports -def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): +def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType], **extra_args): """ Returns the validation results of a dataset based on task type. @@ -1167,15 +1220,33 @@ def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): """ + few_samples_thr = extra_args['few_samples_thr'] + imbalance_ratio_thr = extra_args['imbalance_ratio_thr'] + far_from_mean_thr = extra_args['far_from_mean_thr'] + dominance_ratio_thr = extra_args['dominance_ratio_thr'] + topk_bins = extra_args['topk_bins'] + validation_results = {} task_type = parse_str_enum_value(task_type, TaskType) if task_type == TaskType.classification: - validator = ClassificationValidator() + validator = ClassificationValidator(few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, + topk_bins=topk_bins) elif task_type == TaskType.detection: - validator = DetectionValidator() + validator = DetectionValidator(few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, + topk_bins=topk_bins) elif task_type == TaskType.segmentation: - validator = SegmentationValidator() + validator = SegmentationValidator(few_samples_thr=few_samples_thr, + imbalance_ratio_thr=imbalance_ratio_thr, + far_from_mean_thr=far_from_mean_thr, + dominance_ratio_thr=dominance_ratio_thr, + topk_bins=topk_bins) if not isinstance(dataset, IDataset): raise TypeError("Invalid dataset type '%s'" % type(dataset)) diff --git a/tests/test_validator.py b/tests/test_validator.py index 7229efe741..041e7e66b9 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -31,11 +31,11 @@ def setUpClass(cls): 'a': 1, 'b': 2, }), Mask(id=2, label=0, attributes={'a': 1, 'b': 2}, - image=np.array([[0, 0, 0, 0, 0], - [0, 0, 1, 1, 1], - [0, 0, 1, 1, 1], - [0, 0, 1, 1, 1], - [0, 0, 1, 1, 1], + image=np.array([[0, 0, 0, 0, 0], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], ])), ]), DatasetItem(id=2, image=np.ones((2, 4, 3)), annotations=[ @@ -79,10 +79,10 @@ def setUpClass(cls): 'a': 2, 'b': 2, }), Mask(id=2, label=1, attributes={'a': 2, 'b': 2}, - image=np.array([[1, 0, 0], - [1, 0, 0], - [1, 0, 0], - [1, 0, 0], + image=np.array([[1, 0, 0], + [1, 0, 0], + [1, 0, 0], + [1, 0, 0], ])), ]), DatasetItem(id=7, image=np.ones((2, 4, 3)), annotations=[ @@ -91,7 +91,7 @@ def setUpClass(cls): 'a': 1, 'b': 2, }), Polygon([1, 2, 1, 5, 5, 5, 5, 2], label=2, id=2, - attributes={'a': 1, 'b': 2, + attributes={'a': 1, 'b': 2, }), ]), DatasetItem(id=8, image=np.ones((2, 4, 3)), annotations=[ @@ -100,10 +100,10 @@ def setUpClass(cls): 'a': 2, 'b': 1, }), Mask(id=2, label=2, attributes={'a': 2, 'b': 1}, - image=np.array([[1, 1, 1], - [1, 1, 1], - [1, 1, 1], - [1, 1, 1], + image=np.array([[1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], ])), ]), ], categories=[[f'label_{i}', None, {'a', 'b', }] @@ -113,7 +113,9 @@ def setUpClass(cls): class TestBaseValidator(TestValidatorTemplate): @classmethod def setUpClass(cls): - cls.validator = _Validator(TaskType.classification) + cls.validator = _Validator(task_type=TaskType.classification, + few_samples_thr=1, imbalance_ratio_thr=50, far_from_mean_thr=5.0, + dominance_ratio_thr=0.8, topk_bins=0.1) def test_generate_reports(self): with self.assertRaises(NotImplementedError): @@ -233,7 +235,7 @@ def test_check_few_samples_in_label(self): stats = { 'label_distribution': { 'defined_labels': { - 'unit': self.validator.DEFAULT_FEW_SAMPLES + 'unit': self.validator.few_samples_thr } } } @@ -247,7 +249,7 @@ def test_check_few_samples_in_label(self): stats = { 'label_distribution': { 'defined_labels': { - 'unit': self.validator.DEFAULT_FEW_SAMPLES + 1 + 'unit': self.validator.few_samples_thr + 1 } } } @@ -263,7 +265,7 @@ def test_check_few_samples_in_attribute(self): with self.subTest('Few Samples'): attr_dets = { 'distribution': { - 'mock': self.validator.DEFAULT_FEW_SAMPLES + 'mock': self.validator.few_samples_thr } } @@ -276,7 +278,7 @@ def test_check_few_samples_in_attribute(self): with self.subTest('No Few Samples Warning'): attr_dets = { 'distribution': { - 'mock': self.validator.DEFAULT_FEW_SAMPLES + 1 + 'mock': self.validator.few_samples_thr + 1 } } @@ -290,7 +292,7 @@ def test_check_imbalanced_labels(self): stats = { 'label_distribution': { 'defined_labels': { - 'unit': self.validator.DEFAULT_IMBALANCE_RATIO, + 'unit': self.validator.imbalance_ratio_thr, 'test': 1 } } @@ -305,7 +307,7 @@ def test_check_imbalanced_labels(self): stats = { 'label_distribution': { 'defined_labels': { - 'unit': self.validator.DEFAULT_IMBALANCE_RATIO - 1, + 'unit': self.validator.imbalance_ratio_thr - 1, 'test': 1 } } @@ -322,7 +324,7 @@ def test_check_imbalanced_attribute(self): with self.subTest('Imbalance'): attr_dets = { 'distribution': { - 'mock': self.validator.DEFAULT_IMBALANCE_RATIO, + 'mock': self.validator.imbalance_ratio_thr, 'mock_1': 1 } } @@ -336,7 +338,7 @@ def test_check_imbalanced_attribute(self): with self.subTest('No Imbalance Warning'): attr_dets = { 'distribution': { - 'mock': self.validator.DEFAULT_IMBALANCE_RATIO - 1, + 'mock': self.validator.imbalance_ratio_thr - 1, 'mock_1': 1 } } @@ -350,7 +352,9 @@ def test_check_imbalanced_attribute(self): class TestClassificationValidator(TestValidatorTemplate): @classmethod def setUpClass(cls): - cls.validator = ClassificationValidator() + cls.validator = ClassificationValidator(few_samples_thr=1, + imbalance_ratio_thr=50, far_from_mean_thr=5.0, + dominance_ratio_thr=0.8, topk_bins=0.1) def test_check_missing_label_annotation(self): stats = { @@ -376,11 +380,13 @@ def test_check_multi_label_annotations(self): class TestDetectionValidator(TestValidatorTemplate): @classmethod def setUpClass(cls): - cls.validator = DetectionValidator() + cls.validator = DetectionValidator(few_samples_thr=1, + imbalance_ratio_thr=50, far_from_mean_thr=5.0, + dominance_ratio_thr=0.8, topk_bins=0.1) def test_check_imbalanced_dist_in_label(self): label_name = 'unittest' - most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + most = int(self.validator.dominance_thr * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -413,7 +419,7 @@ def test_check_imbalanced_dist_in_label(self): def test_check_imbalanced_dist_in_attr(self): label_name = 'unit' attr_name = 'test' - most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + most = int(self.validator.dominance_thr * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -534,11 +540,13 @@ def test_check_far_from_attr_mean(self): class TestSegmentationValidator(TestValidatorTemplate): @classmethod def setUpClass(cls): - cls.validator = SegmentationValidator() + cls.validator = SegmentationValidator(few_samples_thr=1, + imbalance_ratio_thr=50, far_from_mean_thr=5.0, + dominance_ratio_thr=0.8, topk_bins=0.1) def test_check_imbalanced_dist_in_label(self): label_name = 'unittest' - most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + most = int(self.validator.dominance_thr * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -571,7 +579,7 @@ def test_check_imbalanced_dist_in_label(self): def test_check_imbalanced_dist_in_attr(self): label_name = 'unit' attr_name = 'test' - most = int(self.validator.DEFAULT_DOMINANCE_RATIO * 100) + most = int(self.validator.dominance_thr * 100) rest = 100 - most with self.subTest('Imbalanced'): @@ -674,8 +682,16 @@ def test_check_far_from_attr_mean(self): class TestValidateAnnotations(TestValidatorTemplate): + extra_args = { + 'few_samples_thr': 1, + 'imbalance_ratio_thr': 50, + 'far_from_mean_thr': 5.0, + 'dominance_ratio_thr': 0.8, + 'topk_bins': 0.1, + } def test_validate_annotations_classification(self): - actual_results = validate_annotations(self.dataset, 'classification') + actual_results = validate_annotations(self.dataset, 'classification', + **self.extra_args) with self.subTest('Test of statistics', i=0): actual_stats = actual_results['statistics'] @@ -730,7 +746,8 @@ def test_validate_annotations_classification(self): self.assertEqual(actual_summary, expected_summary) def test_validate_annotations_detection(self): - actual_results = validate_annotations(self.dataset, 'detection') + actual_results = validate_annotations(self.dataset, 'detection', + **self.extra_args) with self.subTest('Test of statistics', i=0): actual_stats = actual_results['statistics'] @@ -783,7 +800,8 @@ def test_validate_annotations_detection(self): self.assertEqual(actual_summary, expected_summary) def test_validate_annotations_segmentation(self): - actual_results = validate_annotations(self.dataset, 'segmentation') + actual_results = validate_annotations(self.dataset, 'segmentation', + **self.extra_args) with self.subTest('Test of statistics', i=0): actual_stats = actual_results['statistics'] @@ -838,8 +856,8 @@ def test_validate_annotations_segmentation(self): def test_validate_annotations_invalid_task_type(self): with self.assertRaises(ValueError): - validate_annotations(self.dataset, 'INVALID') + validate_annotations(self.dataset, 'INVALID', **self.extra_args) def test_validate_annotations_invalid_dataset_type(self): with self.assertRaises(TypeError): - validate_annotations(object(), 'classification') + validate_annotations(object(), 'classification', **self.extra_args) From 70e26477efa0b40aa1f4a6ea4c9493504f7ff398 Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Tue, 1 Jun 2021 10:29:26 +0300 Subject: [PATCH 29/40] CI. Move to GitHub actions. (#263) * Moving to GitHub Actions --- .github/workflows/coverage.yml | 30 +++++++++++++++++++++++++++ .github/workflows/main.yml | 29 ++++++++++++++++++++++++++ .travis.yml | 38 ---------------------------------- 3 files changed, 59 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..a3c6d464e1 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,30 @@ +name: Coverage +on: + push: + branches: + - 'develop' +jobs: + coverage_tests_on_python: + strategy: + fail-fast: false + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Installing python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Installing dependencies + run: | + pip install coverage tensorflow + pip install -e ./ + - name: Code instrumentation + run: | + coverage run -m unittest discover -v + coverage run -a datum.py -h + coverage xml + - name: Sending coverage results + run: | + bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml -t ${{ secrets.CODACY_PROJECT_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..5689e72d74 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,29 @@ +name: CI +on: + push: + branches: + - 'master' + pull_request: + branches: + - '*' +jobs: + build_and_tests_on_python: + strategy: + fail-fast: false + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Installing python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Installing dependencies + run: | + pip install tensorflow + pip install -e ./ + - name: Unit testing + run: | + python -m unittest discover -v + datum -h diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6b9f252237..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,38 +0,0 @@ -language: python - -cache: pip - -python: - - '3.6' - - '3.7' - - '3.8' - -matrix: - include: - - dist: xenial - - # measure coverage here - - dist: bionic - python: '3.6' - before_install: - - pip install coverage - script: - - coverage run -m unittest discover -v - - coverage run -a datum.py -h - after_success: - - coverage xml - - bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml - - - dist: bionic - python: '3.7' - - dist: bionic - python: '3.8' - -install: - - pip install -e ./ - - pip install tensorflow - - pip install pandas - -script: - - python -m unittest discover -v - - datum -h \ No newline at end of file From c5bf74b889a84c726cbf25968f5b52a5dce75648 Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Tue, 1 Jun 2021 13:39:27 +0300 Subject: [PATCH 30/40] Sending a coverage report if python3.6 (#264) --- .github/workflows/coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a3c6d464e1..3e5d9911f6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -26,5 +26,6 @@ jobs: coverage run -a datum.py -h coverage xml - name: Sending coverage results + if: matrix.python-version == '3.6' run: | bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml -t ${{ secrets.CODACY_PROJECT_TOKEN }} From 817cda5fb48cc67f251ba80652747126110a5490 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 1 Jun 2021 13:52:25 +0300 Subject: [PATCH 31/40] Rename workflows (#265) * Rename workflows --- .github/workflows/{coverage.yml => health_check.yml} | 2 +- .github/workflows/{main.yml => pr_checks.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{coverage.yml => health_check.yml} (98%) rename .github/workflows/{main.yml => pr_checks.yml} (100%) diff --git a/.github/workflows/coverage.yml b/.github/workflows/health_check.yml similarity index 98% rename from .github/workflows/coverage.yml rename to .github/workflows/health_check.yml index 3e5d9911f6..78f93beda1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/health_check.yml @@ -1,4 +1,4 @@ -name: Coverage +name: Build on: push: branches: diff --git a/.github/workflows/main.yml b/.github/workflows/pr_checks.yml similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/pr_checks.yml From 187503d395985d3ac916d835c82b63e95582d573 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 1 Jun 2021 14:00:55 +0300 Subject: [PATCH 32/40] Update repo config and badge (#266) * Update PR template * Update build status badge --- .github/pull_request_template.md | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d40cfb94da..9708adff80 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -24,17 +24,17 @@ not fully covered by unit tests or manual testing can be complicated. --> https://github.com/openvinotoolkit/datumaro/tree/develop/docs) accordingly - [ ] I have added tests to cover my changes - [ ] I have [linked related issues]( - https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) + https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) ### License - [ ] I submit _my code changes_ under the same [MIT License]( - https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project. + https://github.com/openvinotoolkit/datumaro/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. - [ ] I have updated the license header for each file (see an example below) ```python -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT ``` diff --git a/README.md b/README.md index 559eac4762..43054ea2ac 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Dataset Management Framework (Datumaro) -[![Build Status](https://travis-ci.org/openvinotoolkit/datumaro.svg?branch=develop)](https://travis-ci.org/openvinotoolkit/datumaro) +[![Build status](https://github.com/openvinotoolkit/datumaro/actions/workflows/health_check.yml/badge.svg)](https://github.com/openvinotoolkit/datumaro/actions/workflows/health_check.yml) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/759d2d873b59495aa3d3f8c51b786246)](https://app.codacy.com/gh/openvinotoolkit/datumaro?utm_source=github.com&utm_medium=referral&utm_content=openvinotoolkit/datumaro&utm_campaign=Badge_Grade_Dashboard) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/9511b691ff134e739ea6fc524f7cc760)](https://www.codacy.com/gh/openvinotoolkit/datumaro?utm_source=github.com&utm_medium=referral&utm_content=openvinotoolkit/datumaro&utm_campaign=Badge_Coverage) From a77a2a4c4745a652c09ef7ae44967d1883fca37d Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 1 Jun 2021 17:41:01 +0300 Subject: [PATCH 33/40] Fix deprecation warnings (#270) --- datumaro/cli/commands/explain.py | 2 +- datumaro/components/operations.py | 4 ++-- datumaro/plugins/cifar_format.py | 2 +- datumaro/plugins/coco_format/converter.py | 2 +- datumaro/plugins/coco_format/importer.py | 4 ++-- datumaro/plugins/ndr.py | 18 ++++++++++-------- datumaro/plugins/yolo_format/converter.py | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/datumaro/cli/commands/explain.py b/datumaro/cli/commands/explain.py index 9c3e1d147a..c47ae011fb 100644 --- a/datumaro/cli/commands/explain.py +++ b/datumaro/cli/commands/explain.py @@ -152,7 +152,7 @@ def explain_command(args): for item in dataset: image = item.image.data if image is None: - log.warn( + log.warning( "Dataset item %s does not have image data. Skipping." % \ (item.id)) continue diff --git a/datumaro/components/operations.py b/datumaro/components/operations.py index bfe26aec99..002e76f623 100644 --- a/datumaro/components/operations.py +++ b/datumaro/components/operations.py @@ -1032,8 +1032,8 @@ def _extractor_stats(extractor): for item in extractor: if not (item.has_image and item.image.has_data): available = False - log.warn("Item %s has no image. Image stats won't be computed", - item.id) + log.warning("Item %s has no image, it will be excluded from " + "image stats", item.id) break stats = { diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index e2a2bec12b..b6f3a01a23 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -158,7 +158,7 @@ def apply(self): annotation_dict = {} annotation_dict['filenames'] = filenames annotation_dict['labels'] = labels - annotation_dict['data'] = np.array(data) + annotation_dict['data'] = np.array(data, dtype=object) if len(image_sizes): size = (CifarPath.IMAGE_SIZE, CifarPath.IMAGE_SIZE) # 'image_sizes' isn't included in the standard format, diff --git a/datumaro/plugins/coco_format/converter.py b/datumaro/plugins/coco_format/converter.py index 36781a0396..f7caa56b96 100644 --- a/datumaro/plugins/coco_format/converter.py +++ b/datumaro/plugins/coco_format/converter.py @@ -265,7 +265,7 @@ def save_annotations(self, item): return if not item.has_image: - log.warn("Item '%s': skipping writing instances " + log.warning("Item '%s': skipping writing instances " "since no image info available" % item.id) return h, w = item.image.size diff --git a/datumaro/plugins/coco_format/importer.py b/datumaro/plugins/coco_format/importer.py index 2e8f8a2ac8..8d41376ea8 100644 --- a/datumaro/plugins/coco_format/importer.py +++ b/datumaro/plugins/coco_format/importer.py @@ -88,10 +88,10 @@ def find_sources(path): try: ann_type = CocoTask[ann_type] except KeyError: - log.warn("Skipping '%s': unknown subset " + log.warning("Skipping '%s': unknown subset " "type '%s', the only known are: %s" % \ (subset_path, ann_type, - ', '.join([e.name for e in CocoTask]))) + ', '.join(e.name for e in CocoTask))) continue subset_name = name_parts[1] subsets[subset_name][ann_type] = subset_path diff --git a/datumaro/plugins/ndr.py b/datumaro/plugins/ndr.py index b92945bbad..060cd8047e 100644 --- a/datumaro/plugins/ndr.py +++ b/datumaro/plugins/ndr.py @@ -22,14 +22,16 @@ class NDR(Transform, CliPlugin): """ - Near-duplicated image removal |n - Removes near-duplicated images in subset |n - Example: control number of outputs to 100 after NDR |n - |s|s%(prog)s \ |n - |s|s|s|s--working_subset train \ |n - |s|s|s|s--algorithm gradient \ |n - |s|s|s|s--num_cut 100 \ |n - |s|s|s|s--over_sample random \ |n + Near-duplicated image removal.|n + |n + Removes near-duplicated images in subset|n + |n + Example: apply NDR, return no more than 100 images|n + |s|s%(prog)s|n + |s|s|s|s--working_subset train|n + |s|s|s|s--algorithm gradient|n + |s|s|s|s--num_cut 100|n + |s|s|s|s--over_sample random|n |s|s|s|s--under_sample uniform """ diff --git a/datumaro/plugins/yolo_format/converter.py b/datumaro/plugins/yolo_format/converter.py index fb71b8f172..71f021f0e6 100644 --- a/datumaro/plugins/yolo_format/converter.py +++ b/datumaro/plugins/yolo_format/converter.py @@ -49,7 +49,7 @@ def apply(self): if not subset_name or subset_name == DEFAULT_SUBSET_NAME: subset_name = YoloPath.DEFAULT_SUBSET_NAME elif subset_name not in YoloPath.SUBSET_NAMES: - log.warn("Skipping subset export '%s'. " + log.warning("Skipping subset export '%s'. " "If specified, the only valid names are %s" % \ (subset_name, ', '.join( "'%s'" % s for s in YoloPath.SUBSET_NAMES))) From 81409636fef93b4903191ed871eac8e6abb0267e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 2 Jun 2021 13:35:21 +0300 Subject: [PATCH 34/40] Update RISE docs (#255) * Update rise docs * Update cli help --- README.md | 2 + datumaro/cli/commands/explain.py | 29 +++++++++- docs/user_manual.md | 91 +++++++++++++++++++++++++++++--- 3 files changed, 114 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 43054ea2ac..725c671f1f 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,8 @@ CVAT annotations ---> Publication, statistics etc. - Model integration - Inference (OpenVINO, Caffe, PyTorch, TensorFlow, MxNet, etc.) - Explainable AI ([RISE algorithm](https://arxiv.org/abs/1806.07421)) + - RISE for classification + - RISE for object detection > Check [the design document](docs/design.md) for a full list of features. > Check [the user manual](docs/user_manual.md) for usage instructions. diff --git a/datumaro/cli/commands/explain.py b/datumaro/cli/commands/explain.py index c47ae011fb..dc4256194c 100644 --- a/datumaro/cli/commands/explain.py +++ b/datumaro/cli/commands/explain.py @@ -17,13 +17,38 @@ def build_parser(parser_ctor=argparse.ArgumentParser): parser = parser_ctor(help="Run Explainable AI algorithm", - description="Runs an explainable AI algorithm for a model.") + description=""" + Runs an explainable AI algorithm for a model.|n + |n + This tool is supposed to help an AI developer to debug + a model and a dataset. Basically, it executes inference and + tries to find problems in the trained model - determine decision + boundaries and belief intervals for the classifier.|n + |n + Currently, the only available algorithm is + RISE (https://arxiv.org/pdf/1806.07421.pdf), which runs + inference and then re-runs a model multiple times + on each image to produce a heatmap of activations for + each output of the first inference. As a result, we obtain + few heatmaps, which shows, how image pixels affected + the inference result. This algorithm doesn't require any special + information about the model, but it requires the model to + return all the outputs and confidences. Check the User Manual + for usage examples.|n + Supported scenarios:|n + - RISE for classification|n + - RISE for Object Detection|n + |n + Examples:|n + - Run RISE on an image, display results:|n + |s|s%(prog)s -t path/to/image.jpg -m mymodel rise --max-samples 50 + """, formatter_class=MultilineFormatter) parser.add_argument('-m', '--model', required=True, help="Model to use for inference") parser.add_argument('-t', '--target', default=None, help="Inference target - image, source, project " - "(default: current dir)") + "(default: current project)") parser.add_argument('-o', '--output-dir', dest='save_dir', default=None, help="Directory to save output (default: display only)") diff --git a/docs/user_manual.md b/docs/user_manual.md index 2a83cec341..db7c28f9d5 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1093,7 +1093,8 @@ datum model add \ ``` Interpretation script for an OpenVINO detection model (`convert.py`): -You can find OpenVINO™ model interpreter samples in datumaro/plugins/openvino/samples. [Instruction](datumaro/plugins/openvino/README.md) +You can find OpenVINO model interpreter samples in +`datumaro/plugins/openvino/samples` ([instruction](datumaro/plugins/openvino/README.md)). ``` python from datumaro.components.extractor import * @@ -1182,6 +1183,25 @@ datum diff inference -o diff ### Explain inference +Runs an explainable AI algorithm for a model. + +This tool is supposed to help an AI developer to debug a model and a dataset. +Basically, it executes inference and tries to find problems in the trained +model - determine decision boundaries and belief intervals for the classifier. + +Currently, the only available algorithm is RISE ([article](https://arxiv.org/pdf/1806.07421.pdf)), +which runs inference and then re-runs a model multiple times on each +image to produce a heatmap of activations for each output of the +first inference. As a result, we obtain few heatmaps, which +shows, how image pixels affected the inference result. This algorithm doesn't +require any special information about the model, but it requires the model to +return all the outputs and confidences. The algorighm only supports +classification and detection models. + +The following use cases available: +- RISE for classification +- RISE for object detection + Usage: ``` bash @@ -1200,11 +1220,70 @@ Example: run inference explanation on a single image with visualization ``` bash datum create <...> datum model add mymodel <...> -datum explain \ - -m mymodel \ - -t 'image.png' \ - rise \ - -s 1000 --progressive +datum explain -t image.png -m mymodel \ + rise --max-samples 1000 --progressive +``` + +> Note: this algorithm requires the model to return +> *all* (or a _reasonable_ amount) the outputs and confidences unfiltered, +> i.e. all the `Label` annotations for classification models and +> all the `Bbox`es for detection models. +> You can find examples of the expected model outputs in [`tests/test_RISE.py`](../tests/test_RISE.py) + +For OpenVINO models the output processing script would look like this: + +Classification scenario: + +``` python +from datumaro.components.extractor import * +from datumaro.util.annotation_util import softmax + +def process_outputs(inputs, outputs): + # inputs = model input, array or images, shape = (N, C, H, W) + # outputs = model output, logits, shape = (N, n_classes) + # results = conversion result, [ [ Annotation, ... ], ... ] + results = [] + for input, output in zip(inputs, outputs): + input_height, input_width = input.shape[:2] + confs = softmax(output[0]) + for label, conf in enumerate(confs): + results.append(Label(int(label)), attributes={'score': float(conf)}) + + return results +``` + + +Object Detection scenario: + +``` python +from datumaro.components.extractor import * + +# return a significant number of output boxes to make multiple runs +# statistically correct and meaningful +max_det = 1000 + +def process_outputs(inputs, outputs): + # inputs = model input, array or images, shape = (N, C, H, W) + # outputs = model output, shape = (N, 1, K, 7) + # results = conversion result, [ [ Annotation, ... ], ... ] + results = [] + for input, output in zip(inputs, outputs): + input_height, input_width = input.shape[:2] + detections = output[0] + image_results = [] + for i, det in enumerate(detections): + label = int(det[1]) + conf = float(det[2]) + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + image_results.append(Bbox(x, y, w, h, + label=label, attributes={'score': conf} )) + + results.append(image_results[:max_det]) + + return results ``` ### Transform Project From eb572d969dcc77eb949f3d0a931f5435d32a3f20 Mon Sep 17 00:00:00 2001 From: Slawomir Strehlke Date: Wed, 2 Jun 2021 15:17:15 +0200 Subject: [PATCH 35/40] Pytest related changes (#248) * Tests moved to pytest. Updated CI. Updated requirements. * Updated contribution guide * Added annotations for tests * Updated tests * Added code style guide --- .github/workflows/health_check.yml | 7 +- .github/workflows/pr_checks.yml | 4 +- .gitignore | 5 +- CHANGELOG.md | 1 + CONTRIBUTING.md | 191 ++++++++++++++++++++++++++--- pytest.ini | 3 + requirements.txt | 1 + tests/cli/test_diff.py | 2 + tests/cli/test_voc_format.py | 9 ++ tests/cli/test_yolo_format.py | 8 +- tests/conftest.py | 17 +++ tests/requirements.py | 38 ++++++ tests/test_RISE.py | 3 + tests/test_camvid_format.py | 13 ++ tests/test_cifar_format.py | 9 ++ tests/test_coco_format.py | 53 +++++++- tests/test_command_targets.py | 13 ++ tests/test_config.py | 2 + tests/test_cvat_format.py | 14 ++- tests/test_dataset.py | 39 ++++++ tests/test_datumaro_format.py | 8 ++ tests/test_diff.py | 11 ++ tests/test_icdar_format.py | 14 ++- tests/test_image.py | 5 + tests/test_image_dir_format.py | 6 + tests/test_imagenet_format.py | 8 ++ tests/test_imagenet_txt_format.py | 8 ++ tests/test_images.py | 8 ++ tests/test_labelme_format.py | 8 ++ tests/test_lfw_format.py | 9 ++ tests/test_market1501_format.py | 9 ++ tests/test_masks.py | 9 ++ tests/test_mnist_csv_format.py | 10 ++ tests/test_mnist_format.py | 10 ++ tests/test_mot_format.py | 5 + tests/test_mots_format.py | 6 + tests/test_ndr.py | 13 ++ tests/test_ops.py | 11 ++ tests/test_project.py | 21 ++++ tests/test_sampler.py | 8 ++ tests/test_splitter.py | 20 +++ tests/test_tfrecord_format.py | 13 ++ tests/test_transforms.py | 16 +++ tests/test_util.py | 8 ++ tests/test_validator.py | 35 ++++++ tests/test_vgg_face2_format.py | 10 ++ tests/test_voc_format.py | 26 ++++ tests/test_widerface_format.py | 8 ++ tests/test_yolo_format.py | 10 ++ 49 files changed, 740 insertions(+), 25 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/conftest.py create mode 100644 tests/requirements.py diff --git a/.github/workflows/health_check.yml b/.github/workflows/health_check.yml index 78f93beda1..e721a24d4e 100644 --- a/.github/workflows/health_check.yml +++ b/.github/workflows/health_check.yml @@ -18,13 +18,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Installing dependencies run: | - pip install coverage tensorflow + pip install tensorflow pytest pytest-cov pip install -e ./ - name: Code instrumentation run: | - coverage run -m unittest discover -v - coverage run -a datum.py -h - coverage xml + pytest -v --cov --cov-report xml:coverage.xml + datum.py -h - name: Sending coverage results if: matrix.python-version == '3.6' run: | diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 5689e72d74..0201fb6f69 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -21,9 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Installing dependencies run: | - pip install tensorflow + pip install tensorflow pytest pip install -e ./ - name: Unit testing run: | - python -m unittest discover -v + pytest -v datum -h diff --git a/.gitignore b/.gitignore index 79362022a9..78d0010336 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,7 @@ coverage.xml cover/ # Sphinx documentation -docs/_build/ \ No newline at end of file +docs/_build/ + +#Pycharm config files +.idea/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f1f14d4d..859379c6b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allowed arbitrary subset count and names in classification and detection splitters () - Annotation-less dataset elements are now participate in subset splitting () - Classification task in LFW dataset format () +- Testing is now performed with pytest instead of unittest () ### Deprecated - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4822c00075..5d77871820 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,12 @@ ## Table of Contents +- [Design document](docs/design.md) +- [Developer guide](docs/developer_guide.md) - [Installation](#installation) - [Usage](#usage) +- [Code style](#code-style) +- [Development environment](#environment) - [Testing](#testing) -- [Design](#design-and-code-structure) ## Installation @@ -64,15 +67,73 @@ python datum.py --help import datumaro ``` -## Testing +## Code style + +Try to be readable and consistent with the existing codebase. + +The project mostly follows PEP8 with little differences. +Continuation lines have a standard indentation step by default, +or any other, if it improves readability. For long conditionals use 2 steps. +No trailing whitespaces, 80 characters per line. + +Example: + +```python +def do_important_work(parameter1, parameter2, parameter3, + option1=None, option2=None, option3=None) -> str: + """ + Optional description. Mandatory for API. + Use comments for implementation specific information, use docstrings + to give information to user / developer. + + Returns: status (str) - Possible values: 'done', 'failed' + """ + + ... do stuff ... + + # Use +1 level of indentation for continuation lines + variable_with_a_long_but_meaningful_name = \ + function_with_a_long_but_meaningful_name(arg1, arg2, arg3, + kwarg1=value_with_a_long_name, kwarg2=value_with_a_long_name) + + # long conditions, loops, with etc. also use +1 level of indentation + if condition1 and long_condition2 or \ + not condition3 and condition4 and condition5 or \ + condition6 and condition7: + + ... do other stuff ... + + elif other_conditions: + + ... some other things ... + + # in some cases special formatting can improve code readability + specific_case_formatting = np.array([ + [0, 1, 1, 0], + [1, 1, 0, 0], + [1, 1, 0, 1], + ], dtype=np.int32) + + return status +``` + +## Environment + +The recommended editor is VS Code with the Python language plugin. + +## Testing It is expected that all Datumaro functionality is covered and checked by unit tests. Tests are placed in `tests/` directory. +Currently, we use [`pytest`](https://docs.pytest.org/) for testing, but we +also compatible with `unittest`. To run tests use: ``` bash -python -m unittest discover -s tests +pytest -v +# or +python -m pytest -v ``` If you're working inside of a CVAT environment, you can also use: @@ -81,19 +142,121 @@ If you're working inside of a CVAT environment, you can also use: python manage.py test datumaro/ ``` -## Design and code structure -- [Design document](docs/design.md) -- [Developer guide](docs/developer_guide.md) +### Test cases -## Code style +### Test marking -Try to be readable and consistent with the existing codebase. -The project mostly follows PEP8 with little differences. -Continuation lines have a standard indentation step by default, -or any other, if it improves readability. For long conditionals use 2 steps. -No trailing whitespaces, 80 characters per line. +For better integration with CI and requirements tracking, +we use special annotations for tests. -## Environment +A test needs to marked with a requirement it is related to. To mark a test, use: + +```python +from unittest import TestCase +from .requirements import Requirements, mark_requirement + +class MyTests(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) + def test_my_requirement(self): + ... do stuff ... +``` + +Such marking will apply markings from the requirement specified. +They can be overriden for a specific test: + +```python +import pytest + + @pytest.mark.proirity_low + @mark_requirement(Requirements.DATUM_GENERAL_REQ) + def test_my_requirement(self): + ... do stuff ... +``` + +#### Requirements + +Requirements and other links need to be added to [`tests/requirements.py`](tests/requirements.py): + +```python +DATUM_244 = "Add Snyk integration" +DATUM_BUG_219 = "Return format is not uniform" +``` + +```python +# Fully defined in GitHub issues: +@pytest.mark.reqids(Requirements.DATUM_244, Requirements.DATUM_333) + +# And defined ony other way: +@pytest.mark.reqids(Requirements.DATUM_GENERAL_REQ) +``` + + +##### Available annotations for tests and requirements + +Markings are defined in [`tests/conftest.py`](tests/conftest.py). + +**A list of requirements and bugs** +```python +@pytest.mark.requids(Requirements.DATUM_123) +@pytest.mark.bugs(Requirements.DATUM_BUG_456) +``` + +**A priority** +```python +@pytest.mark.priority_low +@pytest.mark.priority_medium +@pytest.mark.priority_high +``` + +**Component** +The marking used for indication of different system components + +```python +@pytest.mark.components(DatumaroComponent.Datumaro) +``` + +**Skipping tests** + +```python +@pytest.mark.skip(SkipMessages.NOT_IMPLEMENTED) +``` + +**Parametrized runs** + +Parameters are used for running the same test with different parameters e.g. + +```python +@pytest.mark.parametrize("numpy_array, batch_size", [ + (np.zeros([2]), 0), + (np.zeros([2]), 1), + (np.zeros([2]), 2), + (np.zeros([2]), 5), + (np.zeros([5]), 2), +]) +``` + +### Test documentation + +Tests are documented with docstrings. Test descriptions must contain +the following: sections: `Description`, `Expected results` and `Steps`. + +```python +def test_can_convert_polygons_to_mask(self): + """ + Description: + Ensure that the dataset polygon annotation can be properly converted + into dataset segmentation mask. + + Expected results: + Dataset segmentation mask converted from dataset polygon annotation + is equal to an expected mask. -The recommended editor is VS Code with the Python plugin. \ No newline at end of file + Steps: + 1. Prepare dataset with polygon annotation + 2. Prepare dataset with expected mask segmentation mode + 3. Convert source dataset to target, with conversion of annotation + from polygon to mask. + 4. Verify that resulting segmentation mask is equal to the expected mask. + """ +``` \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..976c84825b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +python_classes = +python_functions = diff --git a/requirements.txt b/requirements.txt index 5cfc7dd4f2..c067c3b3e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ PyYAML>=5.3.1 scikit-image>=0.15.0 tensorboardX>=1.8 pandas>=1.1.5 +pytest>=5.3.5 \ No newline at end of file diff --git a/tests/cli/test_diff.py b/tests/cli/test_diff.py index 591b117119..96bf97fa25 100644 --- a/tests/cli/test_diff.py +++ b/tests/cli/test_diff.py @@ -15,9 +15,11 @@ ) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir +from ..requirements import Requirements, mark_requirement class DiffTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_compare_projects(self): # just a smoke test label_categories1 = LabelCategories.from_iterable(['x', 'a', 'b', 'y']) mask_categories1 = MaskCategories.make_default(len(label_categories1)) diff --git a/tests/cli/test_voc_format.py b/tests/cli/test_voc_format.py index 8a686e4f58..a707a46513 100644 --- a/tests/cli/test_voc_format.py +++ b/tests/cli/test_voc_format.py @@ -9,6 +9,7 @@ from datumaro.components.extractor import Bbox, Mask, Image, Label from datumaro.cli.__main__ import main from datumaro.util.test_utils import TestDir, compare_datasets +from ..requirements import Requirements, mark_requirement DUMMY_DATASETS_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], 'tests', 'assets', 'voc_dataset') @@ -16,6 +17,7 @@ def run(test, *args, expected_code=0): test.assertEqual(expected_code, main(args), str(args)) + class VocIntegrationScenarios(TestCase): def _test_can_save_and_load(self, project_path, source_path, source_dataset, dataset_format, result_path=None, label_map=None): @@ -30,6 +32,7 @@ def _test_can_save_and_load(self, project_path, source_path, source_dataset, target_dataset = Dataset.import_from(result_path, dataset_format) compare_datasets(self, source_dataset, target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_preparing_dataset_for_train_model(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='c', subset='train', @@ -80,6 +83,7 @@ def test_preparing_dataset_for_train_model(self): parsed_dataset = Dataset.import_from(export_path, format='voc') compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_convert_to_voc_format(self): label_map = OrderedDict(('label_' + str(i), [None, [], []]) for i in range(10)) label_map['background'] = [None, [], []] @@ -122,6 +126,7 @@ def test_convert_to_voc_format(self): parsed_dataset = Dataset.import_from(voc_export, format='voc') compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_voc_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', @@ -164,6 +169,7 @@ def test_can_save_and_load_voc_dataset(self): self._test_can_save_and_load(test_dir, voc_dir, source_dataset, 'voc', label_map='voc') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_voc_layout_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', @@ -196,6 +202,7 @@ def test_can_save_and_load_voc_layout_dataset(self): self._test_can_save_and_load(test_dir, voc_layout_path, source_dataset, 'voc_layout', result_path=result_voc_path, label_map='voc') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_voc_detect_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', @@ -234,6 +241,7 @@ def test_can_save_and_load_voc_detect_dataset(self): self._test_can_save_and_load(test_dir, voc_detection_path, source_dataset, 'voc_detection', result_path=result_voc_path, label_map='voc') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_voc_segmentation_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', @@ -252,6 +260,7 @@ def test_can_save_and_load_voc_segmentation_dataset(self): self._test_can_save_and_load(test_dir, voc_segm_path, source_dataset, 'voc_segmentation', result_path=result_voc_path, label_map='voc') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_voc_action_dataset(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', diff --git a/tests/cli/test_yolo_format.py b/tests/cli/test_yolo_format.py index d5242daefa..2ff047a010 100644 --- a/tests/cli/test_yolo_format.py +++ b/tests/cli/test_yolo_format.py @@ -9,11 +9,13 @@ AnnotationType, Bbox) from datumaro.util.test_utils import TestDir, compare_datasets import datumaro.plugins.voc_format.format as VOC +from ..requirements import Requirements, mark_requirement def run(test, *args, expected_code=0): test.assertEqual(expected_code, main(args), str(args)) class YoloIntegrationScenarios(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_yolo_dataset(self): target_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -38,6 +40,7 @@ def test_can_save_and_load_yolo_dataset(self): parsed_dataset = Dataset.import_from(export_dir, format='yolo') compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_export_mot_as_yolo(self): target_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -61,6 +64,7 @@ def test_can_export_mot_as_yolo(self): parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_convert_voc_to_yolo(self): target_dataset = Dataset.from_iterable([ DatasetItem(id='2007_000001', subset='train', @@ -84,6 +88,7 @@ def test_can_convert_voc_to_yolo(self): parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_ignore_non_supported_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='img1', subset='test', @@ -123,6 +128,7 @@ def test_can_ignore_non_supported_subsets(self): parsed_dataset = Dataset.import_from(yolo_dir, format='yolo') compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_delete_labels_from_yolo_dataset(self): target_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -154,4 +160,4 @@ def test_can_delete_labels_from_yolo_dataset(self): '-f', 'yolo', '--', '--save-image') parsed_dataset = Dataset.import_from(export_dir, format='yolo') - compare_datasets(self, target_dataset, parsed_dataset) \ No newline at end of file + compare_datasets(self, target_dataset, parsed_dataset) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..75e399ee31 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +def pytest_configure(config): + # register additional markers + config.addinivalue_line("markers", "unit: mark a test as unit test") + config.addinivalue_line("markers", "component: mark a test a component test") + config.addinivalue_line("markers", "cli: mark a test a CLI test") + + config.addinivalue_line("markers", "priority_low: mark a test as low priority") + config.addinivalue_line("markers", "priority_medium: mark a test as medium priority") + config.addinivalue_line("markers", "priority_high: mark a test as high priority") + + config.addinivalue_line("markers", "components(ids): link a test with a component") + config.addinivalue_line("markers", "reqids(ids): link a test with a requirement") + config.addinivalue_line("markers", "bugs(ids): link a test with a bug") diff --git a/tests/requirements.py b/tests/requirements.py new file mode 100644 index 0000000000..9bc8c1ad45 --- /dev/null +++ b/tests/requirements.py @@ -0,0 +1,38 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import pytest + + +def mark_requirement(requirement): + def wrapper(test_func): + @pytest.mark.components(DatumaroComponent.Datumaro) + @pytest.mark.component + @pytest.mark.priority_medium + @pytest.mark.reqids(requirement) + def test_wrapper(*args, **kwargs): + return test_func(*args, **kwargs) + return test_wrapper + return wrapper + + +class DatumaroComponent: + Datumaro = "datumaro" + + +class Requirements: + # Exact requirements + DATUM_GENERAL_REQ = "Datumaro general requirement" + + # GitHub issues (not bugs) + # https://github.com/openvinotoolkit/datumaro/issues + DATUM_244 = "Add Snyk integration" + + # GitHub issues (bugs) + # https://github.com/openvinotoolkit/datumaro/issues + DATUM_BUG_219 = "Return format is not uniform" + + +class SkipMessages: + NOT_IMPLEMENTED = "NOT IMPLEMENTED" diff --git a/tests/test_RISE.py b/tests/test_RISE.py index 04772287f4..b32cc0b945 100644 --- a/tests/test_RISE.py +++ b/tests/test_RISE.py @@ -6,9 +6,11 @@ from datumaro.components.extractor import Label, Bbox from datumaro.components.launcher import Launcher from datumaro.components.algorithms.rise import RISE +from .requirements import Requirements, mark_requirement class RiseTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_rise_can_be_applied_to_classification_model(self): class TestLauncher(Launcher): def __init__(self, class_count, roi, **kwargs): @@ -57,6 +59,7 @@ def _process(self, image): hrest_den = (h_sum - roi_sum) / (h_area - roi_area) self.assertLess(hrest_den, roi_den) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_rise_can_be_applied_to_detection_model(self): ROI = namedtuple('ROI', ['threshold', 'x', 'y', 'w', 'h', 'label']) diff --git a/tests/test_camvid_format.py b/tests/test_camvid_format.py index 9bf3b1b234..73df7b2b59 100644 --- a/tests/test_camvid_format.py +++ b/tests/test_camvid_format.py @@ -12,9 +12,11 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) +from .requirements import Requirements, mark_requirement class CamvidFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_write_and_parse_labelmap(self): src_label_map = Camvid.CamvidLabelMap @@ -35,6 +37,7 @@ def categories(self): return Camvid.make_camvid_categories() class CamvidImportTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='0001TP_008550', subset='test', @@ -73,16 +76,19 @@ def test_can_import(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_camvid(self): self.assertTrue(CamvidImporter.detect(DUMMY_DATASET_DIR)) class CamvidConverterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def _test_save_and_load(self, source_dataset, converter, test_dir, target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='camvid', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_camvid_segm(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -100,6 +106,7 @@ def __iter__(self): partial(CamvidConverter.convert, label_map='camvid'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_camvid_segm_unpainted(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -127,6 +134,7 @@ def __iter__(self): label_map='camvid', apply_colormap=False), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -146,6 +154,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(CamvidConverter.convert, label_map='camvid'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -162,6 +171,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(CamvidConverter.convert, label_map='camvid'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_with_no_masks(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -176,6 +186,7 @@ def __iter__(self): partial(CamvidConverter.convert, label_map='camvid'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_source_labelmap_undefined(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -211,6 +222,7 @@ def categories(self): partial(CamvidConverter.convert, label_map='source'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_source_labelmap_defined(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -245,6 +257,7 @@ def categories(self): partial(CamvidConverter.convert, label_map='source'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): class SrcExtractor(TestExtractorBase): def __iter__(self): diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py index 627c0065ab..480d795954 100644 --- a/tests/test_cifar_format.py +++ b/tests/test_cifar_format.py @@ -8,9 +8,11 @@ from datumaro.plugins.cifar_format import CifarConverter, CifarImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class CifarFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='image_2', subset='test', @@ -33,6 +35,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_without_saving_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a', subset='train_1', @@ -53,6 +56,7 @@ def test_can_save_and_load_without_saving_images(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_different_image_size(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='image_1', @@ -75,6 +79,7 @@ def test_can_save_and_load_with_different_image_size(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id="кириллица с пробелом", @@ -90,6 +95,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -105,6 +111,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_empty_image(self): dataset = Dataset.from_iterable([ DatasetItem(id='a', annotations=[Label(0)]), @@ -121,6 +128,7 @@ def test_can_save_and_load_empty_image(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'cifar_dataset') class CifarImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='image_1', subset='train_1', @@ -145,5 +153,6 @@ def test_can_import(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(CifarImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_coco_format.py b/tests/test_coco_format.py index 16babd93df..c1b033d4f4 100644 --- a/tests/test_coco_format.py +++ b/tests/test_coco_format.py @@ -24,11 +24,13 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) - +from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'coco_dataset') + class CocoImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_instances(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='000000000001', image=np.ones((10, 5, 3)), @@ -51,6 +53,7 @@ def test_can_import_instances(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_captions(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -74,6 +77,7 @@ def test_can_import_captions(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_labels(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -88,6 +92,7 @@ def test_can_import_labels(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_points(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -133,6 +138,7 @@ def test_can_import_points(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_image_info(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=Image(path='1.jpg', size=(10, 15)), @@ -144,6 +150,7 @@ def test_can_import_image_info(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_panoptic(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='000000000001', @@ -164,6 +171,7 @@ def test_can_import_panoptic(self): compare_datasets(self, expected_dataset, dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_stuff(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='000000000001', image=np.ones((10, 5, 3)), @@ -183,6 +191,7 @@ def test_can_import_stuff(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(CocoImporter.detect( osp.join(DUMMY_DATASET_DIR, 'coco_instances'))) @@ -194,6 +203,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='coco', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_captions(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -216,6 +226,7 @@ def test_can_save_and_load_captions(self): self._test_save_and_load(expected_dataset, CocoCaptionsConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_instances(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((4, 4, 3)), @@ -296,6 +307,7 @@ def test_can_save_and_load_instances(self): CocoInstancesConverter.convert, test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_panoptic(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((4, 4, 3)), @@ -338,6 +350,7 @@ def test_can_save_and_load_panoptic(self): partial(CocoPanopticConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_stuff(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((4, 4, 3)), @@ -369,6 +382,7 @@ def test_can_save_and_load_stuff(self): self._test_save_and_load(dataset, CocoStuffConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_merge_polygons_on_loading(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((6, 10, 3)), @@ -406,6 +420,7 @@ def test_can_merge_polygons_on_loading(self): importer_args={'merge_instance_polygons': True}, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_crop_covered_segments(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 5, 3)), @@ -449,7 +464,24 @@ def test_can_crop_covered_segments(self): partial(CocoInstancesConverter.convert, crop_covered=True), test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_convert_polygons_to_mask(self): + """ + Description: + Ensure that the dataset polygon annotation can be properly converted into dataset segmentation mask. + + Expected results: + Dataset segmentation mask converted from dataset polygon annotation is equal to expected mask. + + Steps: + 1. Prepare dataset with polygon annotation (source dataset) + 2. Prepare dataset with expected mask segmentation mode (target dataset) + 3. Convert source dataset to target, with conversion of annotation from polygon to mask. Verify that result + segmentation mask is equal to expected mask. + + """ + + # 1. Prepare dataset with polygon annotation (source dataset) source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((6, 10, 3)), annotations=[ @@ -461,6 +493,7 @@ def test_can_convert_polygons_to_mask(self): ), ], categories=[str(i) for i in range(10)]) + # 2. Prepare dataset with expected mask segmentation mode (target dataset) target_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((6, 10, 3)), annotations=[ @@ -480,11 +513,14 @@ def test_can_convert_polygons_to_mask(self): ), ], categories=[str(i) for i in range(10)]) + # 3. Convert source dataset to target, with conversion of annotation from polygon to mask. Verify that result + # segmentation mask is equal to expected mask. with TestDir() as test_dir: self._test_save_and_load(source_dataset, partial(CocoInstancesConverter.convert, segmentation_mode='mask'), test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_convert_masks_to_polygons(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 10, 3)), @@ -522,6 +558,7 @@ def test_can_convert_masks_to_polygons(self): test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_images(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', attributes={'id': 1}), @@ -538,6 +575,7 @@ def test_can_save_and_load_images(self): self._test_save_and_load(expected_dataset, CocoImageInfoConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='train', @@ -548,6 +586,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): self._test_save_and_load(expected_dataset, CocoImageInfoConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_labels(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -561,6 +600,7 @@ def test_can_save_and_load_labels(self): self._test_save_and_load(expected_dataset, CocoLabelsConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_keypoints(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.zeros((5, 5, 3)), @@ -636,6 +676,7 @@ def test_can_save_and_load_keypoints(self): CocoPersonKeypointsConverter.convert, test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): test_dataset = Dataset.from_iterable([ DatasetItem(id=1, attributes={'id': 1}), @@ -646,6 +687,7 @@ def test_can_save_dataset_with_no_subsets(self): self._test_save_and_load(test_dataset, CocoConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_image_info(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=Image(path='1.jpg', size=(10, 15)), @@ -656,6 +698,7 @@ def test_can_save_dataset_with_image_info(self): self._test_save_and_load(expected_dataset, CocoImageInfoConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3)), @@ -671,6 +714,7 @@ def test_relative_paths(self): partial(CocoImageInfoConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -684,6 +728,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): partial(CocoImageInfoConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_preserve_coco_ids(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), @@ -695,6 +740,7 @@ def test_preserve_coco_ids(self): partial(CocoImageInfoConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_annotation_attributes(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.ones((4, 2, 3)), annotations=[ @@ -707,6 +753,7 @@ def test_annotation_attributes(self): self._test_save_and_load(expected_dataset, CocoConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_auto_annotation_ids(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=2, image=np.ones((4, 2, 3)), annotations=[ @@ -725,6 +772,7 @@ def test_auto_annotation_ids(self): self._test_save_and_load(source_dataset, CocoConverter.convert, test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_reindex(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=2, image=np.ones((4, 2, 3)), annotations=[ @@ -744,6 +792,7 @@ def test_reindex(self): partial(CocoConverter.convert, reindex=True), test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_images_in_single_dir(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((2, 4, 3)), @@ -757,6 +806,7 @@ def test_can_save_images_in_single_dir(self): test_dir, require_images=True) self.assertTrue(osp.isfile(osp.join(test_dir, 'images', '1.jpg'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_images_in_separate_dirs(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((2, 4, 3)), @@ -771,6 +821,7 @@ def test_can_save_images_in_separate_dirs(self): self.assertTrue(osp.isfile(osp.join( test_dir, 'images', 'train', '1.jpg'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset diff --git a/tests/test_command_targets.py b/tests/test_command_targets.py index 5b8a69f318..f5241f1889 100644 --- a/tests/test_command_targets.py +++ b/tests/test_command_targets.py @@ -8,9 +8,11 @@ ImageTarget, SourceTarget from datumaro.util.image import save_image from datumaro.util.test_utils import TestDir +from .requirements import Requirements, mark_requirement class CommandTargetsTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_image_false_when_no_file(self): target = ImageTarget() @@ -18,6 +20,7 @@ def test_image_false_when_no_file(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_image_false_when_false(self): with TestDir() as test_dir: path = osp.join(test_dir, 'test.jpg') @@ -30,6 +33,7 @@ def test_image_false_when_false(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_image_true_when_true(self): with TestDir() as test_dir: path = osp.join(test_dir, 'test.jpg') @@ -41,6 +45,7 @@ def test_image_true_when_true(self): self.assertTrue(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_false_when_no_file(self): target = ProjectTarget() @@ -48,6 +53,7 @@ def test_project_false_when_no_file(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_false_when_no_name(self): target = ProjectTarget(project=Project()) @@ -55,6 +61,7 @@ def test_project_false_when_no_name(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_true_when_project_file(self): with TestDir() as test_dir: path = osp.join(test_dir, 'test.jpg') @@ -66,6 +73,7 @@ def test_project_true_when_project_file(self): self.assertTrue(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_true_when_project_name(self): project_name = 'qwerty' project = Project({ @@ -77,6 +85,7 @@ def test_project_true_when_project_name(self): self.assertTrue(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_false_when_not_project_name(self): project_name = 'qwerty' project = Project({ @@ -88,6 +97,7 @@ def test_project_false_when_not_project_name(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_false_when_not_project_file(self): with TestDir() as test_dir: path = osp.join(test_dir, 'test.jpg') @@ -100,6 +110,7 @@ def test_project_false_when_not_project_file(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_source_false_when_no_project(self): target = SourceTarget() @@ -107,6 +118,7 @@ def test_source_false_when_no_project(self): self.assertFalse(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_source_true_when_source_exists(self): source_name = 'qwerty' project = Project() @@ -117,6 +129,7 @@ def test_source_true_when_source_exists(self): self.assertTrue(status) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_source_false_when_source_doesnt_exist(self): source_name = 'qwerty' project = Project() diff --git a/tests/test_config.py b/tests/test_config.py index 32332b3545..2fee6b2379 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,11 @@ from unittest import TestCase from datumaro.components.config import Config, DictConfig, SchemaBuilder +from .requirements import Requirements, mark_requirement class ConfigTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_produce_multilayer_config_from_dict(self): schema_low = SchemaBuilder() \ .add('options', dict) \ diff --git a/tests/test_cvat_format.py b/tests/test_cvat_format.py index 12d4b82205..d95ed26304 100644 --- a/tests/test_cvat_format.py +++ b/tests/test_cvat_format.py @@ -14,7 +14,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) - +from .requirements import Requirements, mark_requirement DUMMY_IMAGE_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'cvat_dataset', 'for_images') @@ -23,12 +23,15 @@ 'assets', 'cvat_dataset', 'for_video') class CvatImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_image(self): self.assertTrue(CvatImporter.detect(DUMMY_IMAGE_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_video(self): self.assertTrue(CvatImporter.detect(DUMMY_VIDEO_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_load_image(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='img0', subset='train', @@ -61,6 +64,7 @@ def test_can_load_image(self): compare_datasets(self, expected_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_load_video(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='frame_000010', subset='annotations', @@ -147,6 +151,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='cvat', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): src_label_cat = LabelCategories(attributes={'occluded', 'common'}) for i in range(10): @@ -237,6 +242,7 @@ def test_can_save_and_load(self): partial(CvatConverter.convert, save_images=True), test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_allow_undeclared_attrs(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, annotations=[ @@ -260,6 +266,7 @@ def test_can_allow_undeclared_attrs(self): partial(CvatConverter.convert, allow_undeclared_attrs=True), test_dir, target_dataset=target_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3))), @@ -281,6 +288,7 @@ def test_relative_paths(self): partial(CvatConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): label_categories = LabelCategories(attributes={'occluded'}) for i in range(10): @@ -314,6 +322,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(CvatConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem('q/1', image=Image(path='q/1.JPEG', @@ -331,6 +340,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): self.assertTrue(osp.isfile( osp.join(test_dir, 'images', 'a', 'b', 'c', '2.bmp'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_preserve_frame_ids(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), @@ -341,6 +351,7 @@ def test_preserve_frame_ids(self): self._test_save_and_load(expected_dataset, CvatConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_reindex(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), @@ -357,6 +368,7 @@ def test_reindex(self): partial(CvatConverter.convert, reindex=True), test_dir, target_dataset=expected_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset diff --git a/tests/test_dataset.py b/tests/test_dataset.py index cb8a776c02..f8f7f0a085 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -15,9 +15,11 @@ LabelCategories, AnnotationType, Transform) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class DatasetTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_create_from_extractors(self): class SrcExtractor1(Extractor): def __iter__(self): @@ -56,6 +58,7 @@ def __iter__(self): compare_datasets(self, DstExtractor(), dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_create_from_iterable(self): class TestExtractor(Extractor): def __iter__(self): @@ -86,6 +89,7 @@ def categories(self): compare_datasets(self, TestExtractor(), actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_join_datasets_with_empty_categories(self): expected = Dataset.from_iterable([ DatasetItem(1, annotations=[ @@ -111,6 +115,7 @@ def test_can_join_datasets_with_empty_categories(self): compare_datasets(self, expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ Label(2) ]), @@ -123,6 +128,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, loaded_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): env = Environment() env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} @@ -139,6 +145,7 @@ def test_can_detect(self): self.assertEqual(DEFAULT_FORMAT, detected_format) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_and_import(self): env = Environment() env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} @@ -157,6 +164,7 @@ def test_can_detect_and_import(self): self.assertEqual(imported_dataset.format, DEFAULT_FORMAT) compare_datasets(self, source_dataset, imported_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_export_by_string_format_name(self): env = Environment() env.converters.items = {'qq': env.converters[DEFAULT_FORMAT]} @@ -168,6 +176,7 @@ def test_can_export_by_string_format_name(self): with TestDir() as test_dir: dataset.export(format='qq', save_dir=test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_transform_by_string_name(self): expected = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ Label(2) ], attributes={'qq': 1}), @@ -188,6 +197,7 @@ def transform_item(self, item): compare_datasets(self, expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_join_annotations(self): a = Dataset.from_iterable([ DatasetItem(id=1, subset='train', annotations=[ @@ -215,6 +225,7 @@ def test_can_join_annotations(self): compare_datasets(self, expected, merged) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_cant_join_different_categories(self): s1 = Dataset.from_iterable([], categories=['a', 'b']) s2 = Dataset.from_iterable([], categories=['b', 'a']) @@ -222,6 +233,7 @@ def test_cant_join_different_categories(self): with self.assertRaisesRegex(DatumaroError, "different categories"): Dataset.from_extractors(s1, s2) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_join_datasets(self): s1 = Dataset.from_iterable([ DatasetItem(0), DatasetItem(1) ]) s2 = Dataset.from_iterable([ DatasetItem(1), DatasetItem(2) ]) @@ -233,6 +245,7 @@ def test_can_join_datasets(self): compare_datasets(self, expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset @@ -257,6 +270,7 @@ def test_inplace_save_writes_only_updated_data(self): self.assertFalse(osp.isfile(osp.join(path, 'annotations', 'b.json'))) self.assertTrue(osp.isfile(osp.join(path, 'annotations', 'c.json'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_track_modifications_on_addition(self): dataset = Dataset.from_iterable([ DatasetItem(1), @@ -269,6 +283,7 @@ def test_can_track_modifications_on_addition(self): self.assertTrue(dataset.is_modified) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_track_modifications_on_removal(self): dataset = Dataset.from_iterable([ DatasetItem(1), @@ -281,6 +296,7 @@ def test_can_track_modifications_on_removal(self): self.assertTrue(dataset.is_modified) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_create_patch(self): expected = Dataset.from_iterable([ DatasetItem(2), @@ -315,6 +331,7 @@ def test_can_create_patch(self): compare_datasets(self, expected, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_create_more_precise_patch_when_cached(self): expected = Dataset.from_iterable([ DatasetItem(2), @@ -350,6 +367,7 @@ def test_can_create_more_precise_patch_when_cached(self): compare_datasets(self, expected, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_do_lazy_put_and_remove(self): iter_called = False class TestExtractor(Extractor): @@ -375,6 +393,7 @@ def __iter__(self): self.assertTrue(dataset.is_cache_initialized) self.assertTrue(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_put(self): dataset = Dataset() @@ -382,6 +401,7 @@ def test_can_put(self): self.assertTrue((1, '') in dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_do_lazy_get_on_updated_item(self): iter_called = False class TestExtractor(Extractor): @@ -399,6 +419,7 @@ def __iter__(self): self.assertTrue((2, '') in dataset) self.assertFalse(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_switch_eager_and_lazy_with_cm_global(self): iter_called = False class TestExtractor(Extractor): @@ -415,6 +436,7 @@ def __iter__(self): self.assertTrue(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_switch_eager_and_lazy_with_cm_local(self): iter_called = False class TestExtractor(Extractor): @@ -435,6 +457,7 @@ def __iter__(self): self.assertTrue(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_do_lazy_select(self): iter_called = False class TestExtractor(Extractor): @@ -458,6 +481,7 @@ def __iter__(self): self.assertTrue(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_chain_lazy_transforms(self): iter_called = False class TestExtractor(Extractor): @@ -486,12 +510,14 @@ def transform_item(self, item): self.assertTrue(iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_raises_when_repeated_items_in_source(self): dataset = Dataset.from_iterable([DatasetItem(0), DatasetItem(0)]) with self.assertRaises(RepeatedItemError): dataset.init_cache() + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_check_item_existence(self): dataset = Dataset.from_iterable([ DatasetItem(0, subset='a'), DatasetItem(1) @@ -504,6 +530,7 @@ def test_can_check_item_existence(self): self.assertTrue(1 in dataset) self.assertFalse(0 in dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_put_with_id_override(self): dataset = Dataset.from_iterable([]) @@ -511,6 +538,7 @@ def test_can_put_with_id_override(self): self.assertTrue((2, 'b') in dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_compute_cache_with_empty_source(self): dataset = Dataset.from_iterable([]) dataset.put(DatasetItem(2)) @@ -519,6 +547,7 @@ def test_can_compute_cache_with_empty_source(self): self.assertTrue(2 in dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_cant_do_partial_caching_in_get_when_default(self): iter_called = 0 class TestExtractor(Extractor): @@ -539,6 +568,7 @@ def __iter__(self): self.assertEqual(1, iter_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_do_partial_caching_in_get_when_redefined(self): iter_called = 0 get_called = 0 @@ -566,6 +596,7 @@ def get(self, id, subset=None): #pylint: disable=redefined-builtin self.assertEqual(0, iter_called) self.assertEqual(2, get_called) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_binds_on_save(self): dataset = Dataset.from_iterable([DatasetItem(1)]) @@ -578,6 +609,7 @@ def test_binds_on_save(self): self.assertEqual(dataset.data_path, test_dir) self.assertEqual(dataset.format, DEFAULT_FORMAT) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_flushes_changes_on_save(self): dataset = Dataset.from_iterable([]) dataset.put(DatasetItem(1)) @@ -589,6 +621,7 @@ def test_flushes_changes_on_save(self): self.assertFalse(dataset.is_modified) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_does_not_load_images_on_saving(self): # Issue https://github.com/openvinotoolkit/datumaro/issues/177 # Missing image metadata (size etc.) can lead to image loading on @@ -610,6 +643,7 @@ def test_loader(): class DatasetItemTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ctor_requires_id(self): with self.assertRaises(Exception): # pylint: disable=no-value-for-parameter @@ -617,6 +651,7 @@ def test_ctor_requires_id(self): # pylint: enable=no-value-for-parameter @staticmethod + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ctors_with_image(): for args in [ { 'id': 0, 'image': None }, @@ -630,6 +665,7 @@ def test_ctors_with_image(): class DatasetFilterTest(TestCase): @staticmethod + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_item_representations(): item = DatasetItem(id=1, subset='subset', path=['a', 'b'], image=np.ones((5, 4, 3)), @@ -651,6 +687,7 @@ def test_item_representations(): encoded = DatasetItemEncoder.encode(item) DatasetItemEncoder.to_string(encoded) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_item_filter_can_be_applied(self): class TestExtractor(Extractor): def __iter__(self): @@ -663,6 +700,7 @@ def __iter__(self): self.assertEqual(2, len(filtered)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_annotations_filter_can_be_applied(self): class SrcExtractor(Extractor): def __iter__(self): @@ -697,6 +735,7 @@ def __iter__(self): self.assertListEqual(list(filtered), list(DstExtractor())) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_annotations_filter_can_remove_empty_items(self): source = Dataset.from_iterable([ DatasetItem(id=0), diff --git a/tests/test_datumaro_format.py b/tests/test_datumaro_format.py index b3d3a950b2..ae5a88781f 100644 --- a/tests/test_datumaro_format.py +++ b/tests/test_datumaro_format.py @@ -16,6 +16,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets_strict, test_save_and_load) +from .requirements import Requirements, mark_requirement class DatumaroConverterTest(TestCase): @@ -27,6 +28,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, compare=compare_datasets_strict) @property + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset(self): label_categories = LabelCategories(attributes={'a', 'b', 'score'}) for i in range(5): @@ -84,17 +86,20 @@ def test_dataset(self): AnnotationType.points: points_categories, }) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): with TestDir() as test_dir: self._test_save_and_load(self.test_dataset, partial(DatumaroConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): with TestDir() as test_dir: DatumaroConverter.convert(self.test_dataset, save_dir=test_dir) self.assertTrue(DatumaroImporter.detect(test_dir)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): test_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3))), @@ -106,6 +111,7 @@ def test_relative_paths(self): self._test_save_and_load(test_dataset, partial(DatumaroConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): test_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', image=np.ones((4, 2, 3))), @@ -116,6 +122,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(DatumaroConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -129,6 +136,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): partial(DatumaroConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset diff --git a/tests/test_diff.py b/tests/test_diff.py index bb25991ed2..83dfcde1b5 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -6,9 +6,11 @@ from datumaro.components.operations import DistanceComparator, ExactComparator from unittest import TestCase +from .requirements import Requirements, mark_requirement class DistanceComparatorTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_no_bbox_diff_with_same_item(self): detections = 3 anns = [ @@ -31,6 +33,7 @@ def test_no_bbox_diff_with_same_item(self): self.assertLess(iou_thresh, a_bbox.iou(b_bbox)) self.assertEqual(a_bbox.label, b_bbox.label) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_find_bbox_with_wrong_label(self): detections = 3 class_count = 2 @@ -57,6 +60,7 @@ def test_can_find_bbox_with_wrong_label(self): self.assertLess(iou_thresh, a_bbox.iou(b_bbox)) self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_find_missing_boxes(self): detections = 3 class_count = 2 @@ -80,6 +84,7 @@ def test_can_find_missing_boxes(self): self.assertEqual(len(item2.annotations), len(b_greater)) self.assertEqual(0, len(matches)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_no_label_diff_with_same_item(self): detections = 3 anns = [ Label(i) for i in range(detections) ] @@ -92,6 +97,7 @@ def test_no_label_diff_with_same_item(self): self.assertEqual(0, len(b_greater)) self.assertEqual(len(item.annotations), len(matches)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_find_wrong_label(self): item1 = DatasetItem(id=1, annotations=[ Label(0), @@ -111,6 +117,7 @@ def test_can_find_wrong_label(self): self.assertEqual(2, len(b_greater)) self.assertEqual(1, len(matches)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_match_points(self): item1 = DatasetItem(id=1, annotations=[ Points([1, 2, 2, 0, 1, 1], label=0), @@ -132,6 +139,7 @@ def test_can_match_points(self): self.assertEqual(0, len(mismatches)) class ExactComparatorTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_class_comparison(self): a = Dataset.from_iterable([], categories=['a', 'b', 'c']) b = Dataset.from_iterable([], categories=['b', 'c']) @@ -141,6 +149,7 @@ def test_class_comparison(self): self.assertEqual(1, len(errors), errors) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_item_comparison(self): a = Dataset.from_iterable([ DatasetItem(id=1, subset='train'), @@ -159,6 +168,7 @@ def test_item_comparison(self): self.assertEqual({('3', DEFAULT_SUBSET_NAME)}, b_extra_items) self.assertEqual(1, len(errors), errors) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_annotation_comparison(self): a = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ @@ -195,6 +205,7 @@ def test_annotation_comparison(self): self.assertEqual(2, len(unmatched), unmatched) self.assertEqual(0, len(errors), errors) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_image_comparison(self): a = Dataset.from_iterable([ DatasetItem(id=11, image=np.ones((5, 4, 3)), annotations=[ diff --git a/tests/test_icdar_format.py b/tests/test_icdar_format.py index 7559d6fe37..21bd8f0e0f 100644 --- a/tests/test_icdar_format.py +++ b/tests/test_icdar_format.py @@ -16,23 +16,27 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) - +from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'icdar_dataset') class IcdarImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_word_recognition(self): self.assertTrue(IcdarWordRecognitionImporter.detect( osp.join(DUMMY_DATASET_DIR, 'word_recognition'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_text_localization(self): self.assertTrue(IcdarTextLocalizationImporter.detect( osp.join(DUMMY_DATASET_DIR, 'text_localization'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_text_segmentation(self): self.assertTrue(IcdarTextSegmentationImporter.detect( osp.join(DUMMY_DATASET_DIR, 'text_segmentation'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_captions(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='word_1', subset='train', @@ -55,6 +59,7 @@ def test_can_import_captions(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_bboxes(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='img_1', subset='train', @@ -79,6 +84,7 @@ def test_can_import_bboxes(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_masks(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -116,6 +122,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer, importer, target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_captions(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', subset='train', @@ -133,6 +140,7 @@ def test_can_save_and_load_captions(self): partial(IcdarWordRecognitionConverter.convert, save_images=True), test_dir, 'icdar_word_recognition') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_bboxes(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', subset='train', @@ -159,6 +167,7 @@ def test_can_save_and_load_bboxes(self): partial(IcdarTextLocalizationConverter.convert, save_images=True), test_dir, 'icdar_text_localization') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_masks(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', subset='train', @@ -192,6 +201,7 @@ def test_can_save_and_load_masks(self): partial(IcdarTextSegmentationConverter.convert, save_images=True), test_dir, 'icdar_text_segmentation') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_no_subsets(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.ones((8, 8, 3)), @@ -205,6 +215,7 @@ def test_can_save_and_load_with_no_subsets(self): IcdarTextLocalizationConverter.convert, test_dir, 'icdar_text_localization') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', @@ -221,6 +232,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(converter.convert, save_images=True), test_dir, importer, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', diff --git a/tests/test_image.py b/tests/test_image.py index 5f4ef81c4f..1983b711d9 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -6,6 +6,7 @@ import datumaro.util.image as image_module from datumaro.util.test_utils import TestDir +from .requirements import Requirements, mark_requirement class ImageOperationsTest(TestCase): @@ -15,6 +16,7 @@ def setUp(self): def tearDown(self): image_module._IMAGE_BACKEND = self.default_backend + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_save_and_load_backends(self): backends = image_module._IMAGE_BACKENDS for save_backend, load_backend, c in product(backends, backends, [1, 3]): @@ -34,6 +36,7 @@ def test_save_and_load_backends(self): self.assertTrue(np.array_equal(src_image, dst_image), 'save: %s, load: %s' % (save_backend, load_backend)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_encode_and_decode_backends(self): backends = image_module._IMAGE_BACKENDS for save_backend, load_backend, c in product(backends, backends, [1, 3]): @@ -52,11 +55,13 @@ def test_encode_and_decode_backends(self): self.assertTrue(np.array_equal(src_image, dst_image), 'save: %s, load: %s' % (save_backend, load_backend)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_save_image_to_inexistent_dir_raises_error(self): with self.assertRaises(FileNotFoundError): image_module.save_image('some/path.jpg', np.ones((5, 4, 3)), create_dir=False) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_save_image_can_create_dir(self): with TestDir() as test_dir: path = osp.join(test_dir, 'some', 'path.jpg') diff --git a/tests/test_image_dir_format.py b/tests/test_image_dir_format.py index 1b056e8f3e..4cbea707ab 100644 --- a/tests/test_image_dir_format.py +++ b/tests/test_image_dir_format.py @@ -9,9 +9,11 @@ from datumaro.plugins.image_dir_format import ImageDirConverter from datumaro.util.image import Image, save_image from datumaro.util.test_utils import TestDir, compare_datasets, test_save_and_load +from .requirements import Requirements, mark_requirement class ImageDirFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_load(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.ones((10, 6, 3))), @@ -22,6 +24,7 @@ def test_can_load(self): test_save_and_load(self, dataset, ImageDirConverter.convert, test_dir, importer='image_dir', require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3))), @@ -33,6 +36,7 @@ def test_relative_paths(self): test_save_and_load(self, dataset, ImageDirConverter.convert, test_dir, importer='image_dir') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', image=np.ones((4, 2, 3))), @@ -42,6 +46,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): test_save_and_load(self, dataset, ImageDirConverter.convert, test_dir, importer='image_dir') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -54,6 +59,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): test_save_and_load(self, dataset, ImageDirConverter.convert, test_dir, importer='image_dir', require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_custom_extension(self): expected = Dataset.from_iterable([ DatasetItem(id='a/3', image=Image(path='a/3.qq', diff --git a/tests/test_imagenet_format.py b/tests/test_imagenet_format.py index 9a4da64a3f..6735f18123 100644 --- a/tests/test_imagenet_format.py +++ b/tests/test_imagenet_format.py @@ -10,8 +10,11 @@ from datumaro.plugins.imagenet_format import ImagenetConverter, ImagenetImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement + class ImagenetFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', @@ -35,6 +38,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_multiple_labels(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', @@ -57,6 +61,7 @@ def test_can_save_and_load_with_multiple_labels(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id="кириллица с пробелом", @@ -76,6 +81,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='a', image=Image(path='a.JPEG', @@ -95,6 +101,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'imagenet_dataset') class ImagenetImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', @@ -114,5 +121,6 @@ def test_can_import(self): compare_datasets(self, expected_dataset, dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_imagenet(self): self.assertTrue(ImagenetImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_imagenet_txt_format.py b/tests/test_imagenet_txt_format.py index 2c4d231f58..15bfc2b81c 100644 --- a/tests/test_imagenet_txt_format.py +++ b/tests/test_imagenet_txt_format.py @@ -11,9 +11,11 @@ ImagenetTxtConverter, ImagenetTxtImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class ImagenetTxtFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -36,6 +38,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_multiple_labels(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -57,6 +60,7 @@ def test_can_save_and_load_with_multiple_labels(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/c', image=np.zeros((8, 4, 3)), @@ -76,6 +80,7 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): dataset = Dataset.from_iterable([ DatasetItem(id="кириллица с пробелом", @@ -95,6 +100,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='a/1', image=Image(path='a/1.JPEG', @@ -114,6 +120,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'imagenet_txt_dataset') class ImagenetTxtImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', image=np.zeros((8, 6, 3)), @@ -137,5 +144,6 @@ def test_can_import(self): compare_datasets(self, expected_dataset, dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_imagenet(self): self.assertTrue(ImagenetTxtImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_images.py b/tests/test_images.py index a003b8d426..a0c22d607c 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -7,9 +7,11 @@ from datumaro.util.image import (lazy_image, load_image, save_image, \ Image, ByteImage, encode_image) from datumaro.util.image_cache import ImageCache +from .requirements import Requirements, mark_requirement class LazyImageTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_cache_works(self): with TestDir() as test_dir: image = np.ones((100, 100, 3), dtype=np.uint8) @@ -23,6 +25,7 @@ def test_cache_works(self): self.assertFalse(non_caching_loader() is non_caching_loader()) class ImageCacheTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_cache_fifo_displacement(self): capacity = 2 cache = ImageCache(capacity) @@ -39,6 +42,7 @@ def test_cache_fifo_displacement(self): matches = sum([a is b for a, b in zip(first_request, second_request)]) self.assertEqual(matches, len(first_request) - 1) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_global_cache_is_accessible(self): loader = lazy_image(None, loader=lambda p: object()) @@ -47,6 +51,7 @@ def test_global_cache_is_accessible(self): self.assertEqual(ImageCache.get_instance().size(), 1) class ImageTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_lazy_image_shape(self): data = np.ones((5, 6, 3)) @@ -56,6 +61,7 @@ def test_lazy_image_shape(self): self.assertEqual((2, 4), image_lazy.size) self.assertEqual((5, 6), image_eager.size) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ctors(self): with TestDir() as test_dir: path = osp.join(test_dir, 'path.png') @@ -82,6 +88,7 @@ def test_ctors(self): # pylint: enable=pointless-statement class BytesImageTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_lazy_image_shape(self): data = encode_image(np.ones((5, 6, 3)), 'png') @@ -91,6 +98,7 @@ def test_lazy_image_shape(self): self.assertEqual((2, 4), image_lazy.size) self.assertEqual((5, 6), image_eager.size) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ctors(self): with TestDir() as test_dir: path = osp.join(test_dir, 'path.png') diff --git a/tests/test_labelme_format.py b/tests/test_labelme_format.py index f3d2c3f6ef..ad80a9ecf5 100644 --- a/tests/test_labelme_format.py +++ b/tests/test_labelme_format.py @@ -10,6 +10,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) +from .requirements import Requirements, mark_requirement class LabelMeConverterTest(TestCase): @@ -19,6 +20,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='label_me', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='dir1/1', subset='train', @@ -85,6 +87,7 @@ def test_can_save_and_load(self): partial(LabelMeConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='a/1', image=Image(path='a/1.JPEG', @@ -98,6 +101,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): partial(LabelMeConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='train', @@ -123,6 +127,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(LabelMeConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3))), @@ -154,6 +159,7 @@ def test_relative_paths(self): partial(LabelMeConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_to_correct_dir_with_correct_filename(self): dataset = Dataset.from_iterable([ DatasetItem(id='dir/a', image=Image(path='dir/a.JPEG', @@ -172,9 +178,11 @@ def test_can_save_dataset_to_correct_dir_with_correct_filename(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'labelme_dataset') class LabelMeImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(LabelMeImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): img1 = np.ones((77, 102, 3)) * 255 img1[6:32, 7:41] = 0 diff --git a/tests/test_lfw_format.py b/tests/test_lfw_format.py index 2adaae52a4..64a37731a7 100644 --- a/tests/test_lfw_format.py +++ b/tests/test_lfw_format.py @@ -7,9 +7,11 @@ from datumaro.plugins.lfw_format import LfwConverter, LfwImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class LfwFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0_0001', subset='test', @@ -46,6 +48,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_landmarks(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0_0001', @@ -72,6 +75,7 @@ def test_can_save_and_load_with_landmarks(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0_0001', @@ -92,6 +96,7 @@ def test_can_save_and_load_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_no_format_names(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/1', @@ -120,6 +125,7 @@ def test_can_save_and_load_with_no_format_names(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', @@ -139,6 +145,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='a/1', image=Image( @@ -158,9 +165,11 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'lfw_dataset') class LfwImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(LfwImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='name0_0001', subset='test', diff --git a/tests/test_market1501_format.py b/tests/test_market1501_format.py index 9eaaa30fcf..17fc3afeac 100644 --- a/tests/test_market1501_format.py +++ b/tests/test_market1501_format.py @@ -8,9 +8,11 @@ Market1501Importer) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class Market1501FormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='0001_c2s3_000001_00', @@ -45,6 +47,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='0001_c2s3_000001_00', @@ -63,6 +66,7 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', @@ -82,6 +86,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='0001_c2s3_000001_00', @@ -108,6 +113,7 @@ def test_can_save_dataset_with_no_save_images(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image( @@ -133,6 +139,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): compare_datasets(self, expected, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_attributes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='test1', @@ -157,9 +164,11 @@ def test_can_save_dataset_with_no_attributes(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'market1501_dataset') class Market1501ImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(Market1501Importer.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='0001_c2s3_000111_00', diff --git a/tests/test_masks.py b/tests/test_masks.py index 4396966089..1025927a4f 100644 --- a/tests/test_masks.py +++ b/tests/test_masks.py @@ -4,9 +4,11 @@ import datumaro.util.mask_tools as mask_tools from datumaro.components.extractor import CompiledMask +from .requirements import Requirements, mark_requirement class PolygonConversionsTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_mask_can_be_converted_to_polygon(self): mask = np.array([ [0, 1, 1, 1, 0, 1, 1, 1, 1, 0], @@ -24,6 +26,7 @@ def test_mask_can_be_converted_to_polygon(self): self.assertEqual(len(expected), len(computed)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_crop_covered_segments(self): image_size = [7, 7] initial = [ @@ -80,6 +83,7 @@ def _test_mask_to_rle(self, source_mask): self.assertTrue(np.array_equal(source_mask, resulting_mask), '%s\n%s\n' % (source_mask, resulting_mask)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_mask_to_rle_multi(self): cases = [ np.array([ @@ -118,6 +122,7 @@ def test_mask_to_rle_multi(self): self._test_mask_to_rle(case) class ColormapOperationsTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_paint_mask(self): mask = np.zeros((1, 3), dtype=np.uint8) mask[:, 0] = 0 @@ -136,6 +141,7 @@ def test_can_paint_mask(self): self.assertTrue(np.array_equal(expected, actual), '%s\nvs.\n%s' % (expected, actual)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_unpaint_mask(self): colormap = mask_tools.generate_colormap(3) inverse_colormap = mask_tools.invert_colormap(colormap) @@ -155,6 +161,7 @@ def test_can_unpaint_mask(self): self.assertTrue(np.array_equal(expected, actual), '%s\nvs.\n%s' % (expected, actual)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_remap_mask(self): class_count = 10 remap_fn = lambda c: class_count - c @@ -172,6 +179,7 @@ def test_can_remap_mask(self): self.assertTrue(np.array_equal(expected, actual), '%s\nvs.\n%s' % (expected, actual)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_merge_masks(self): masks = [ np.array([0, 2, 4, 0, 0, 1]), @@ -186,6 +194,7 @@ def test_can_merge_masks(self): self.assertTrue(np.array_equal(expected, actual), '%s\nvs.\n%s' % (expected, actual)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_decode_compiled_mask(self): class_idx = 1000 instance_idx = 10000 diff --git a/tests/test_mnist_csv_format.py b/tests/test_mnist_csv_format.py index 17286b90fe..8fd258a8e5 100644 --- a/tests/test_mnist_csv_format.py +++ b/tests/test_mnist_csv_format.py @@ -9,9 +9,11 @@ MnistCsvImporter) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class MnistCsvFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='test', @@ -37,6 +39,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_without_saving_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='train', @@ -57,6 +60,7 @@ def test_can_save_and_load_without_saving_images(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_different_image_size(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, image=np.ones((10, 8)), @@ -77,6 +81,7 @@ def test_can_save_and_load_with_different_image_size(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id="кириллица с пробелом", @@ -95,6 +100,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -113,6 +119,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_empty_image(self): dataset = Dataset.from_iterable([ DatasetItem(id=0, annotations=[Label(0)]), @@ -129,6 +136,7 @@ def test_can_save_and_load_empty_image(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_other_labels(self): dataset = Dataset.from_iterable([ DatasetItem(id=0, image=np.ones((28, 28)), @@ -150,6 +158,7 @@ def test_can_save_and_load_with_other_labels(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mnist_csv_dataset') class MnistCsvImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='test', @@ -181,5 +190,6 @@ def test_can_import(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(MnistCsvImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_mnist_format.py b/tests/test_mnist_format.py index eb5f5299b1..41fe67286f 100644 --- a/tests/test_mnist_format.py +++ b/tests/test_mnist_format.py @@ -8,9 +8,11 @@ from datumaro.plugins.mnist_format import MnistConverter, MnistImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class MnistFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='test', @@ -36,6 +38,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_without_saving_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='train', @@ -56,6 +59,7 @@ def test_can_save_and_load_without_saving_images(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_different_image_size(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=0, image=np.ones((3, 4)), @@ -76,6 +80,7 @@ def test_can_save_and_load_with_different_image_size(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id="кириллица с пробелом", @@ -94,6 +99,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem(id='q/1', image=Image(path='q/1.JPEG', @@ -112,6 +118,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_empty_image(self): dataset = Dataset.from_iterable([ DatasetItem(id=0, annotations=[Label(0)]), @@ -128,6 +135,7 @@ def test_can_save_and_load_empty_image(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_with_other_labels(self): dataset = Dataset.from_iterable([ DatasetItem(id=0, image=np.ones((28, 28)), @@ -149,6 +157,7 @@ def test_can_save_and_load_with_other_labels(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mnist_dataset') class MnistImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=0, subset='test', @@ -180,5 +189,6 @@ def test_can_import(self): compare_datasets(self, expected_dataset, dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(MnistImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_mot_format.py b/tests/test_mot_format.py index e5757d3cd7..6b428b40f8 100644 --- a/tests/test_mot_format.py +++ b/tests/test_mot_format.py @@ -11,6 +11,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) +from .requirements import Requirements, mark_requirement class MotConverterTest(TestCase): @@ -20,6 +21,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='mot_seq', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_bboxes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -98,6 +100,7 @@ def test_can_save_bboxes(self): partial(MotSeqGtConverter.convert, save_images=True), test_dir, target_dataset=target_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem('1', image=Image( @@ -123,9 +126,11 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mot_dataset') class MotImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(MotSeqImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, diff --git a/tests/test_mots_format.py b/tests/test_mots_format.py index 534c01b016..51691a68c4 100644 --- a/tests/test_mots_format.py +++ b/tests/test_mots_format.py @@ -10,6 +10,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) +from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mots_dataset') @@ -21,6 +22,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='mots', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_masks(self): source = Dataset.from_iterable([ DatasetItem(id=1, subset='a', image=np.ones((5, 1)), annotations=[ @@ -67,6 +69,7 @@ def test_can_save_masks(self): partial(MotsPngConverter.convert, save_images=True), test_dir, target_dataset=target) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='a', @@ -81,6 +84,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(MotsPngConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): expected = Dataset.from_iterable([ DatasetItem('q/1', image=Image( @@ -105,9 +109,11 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): test_dir, require_images=True) class MotsImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(MotsImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): target = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((5, 1)), annotations=[ diff --git a/tests/test_ndr.py b/tests/test_ndr.py index cdc32fde73..7180d1f523 100644 --- a/tests/test_ndr.py +++ b/tests/test_ndr.py @@ -7,6 +7,8 @@ LabelCategories, AnnotationType) import datumaro.plugins.ndr as ndr +from .requirements import Requirements, mark_requirement + class NDRTest(TestCase): def _generate_dataset(self, config, num_duplicate, dataset='classification'): @@ -39,6 +41,7 @@ def _generate_dataset(self, config, num_duplicate, dataset='classification'): dataset = Dataset.from_iterable(iterable, categories) return dataset + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_with_error(self): config = { "label1": 100, @@ -96,6 +99,7 @@ def test_ndr_with_error(self): result = ndr.NDR(source, working_subset='train') len(result) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_without_cut(self): config = { "label1": 100, @@ -117,6 +121,7 @@ def test_ndr_without_cut(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_can_use_undersample_uniform(self): config = { "label1": 100, @@ -139,6 +144,7 @@ def test_ndr_can_use_undersample_uniform(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_can_use_undersample_inverse(self): config = { "label1": 100, @@ -161,6 +167,7 @@ def test_ndr_can_use_undersample_inverse(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_can_use_oversample_random(self): config = { "label1": 100, @@ -183,6 +190,7 @@ def test_ndr_can_use_oversample_random(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_can_use_oversample_similarity(self): config = { "label1": 100, @@ -205,6 +213,7 @@ def test_ndr_can_use_oversample_similarity(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_gradient_fails_on_invalid_parameters(self): source = self._generate_dataset({ 'label1': 5 }, 10) @@ -235,6 +244,7 @@ def test_ndr_gradient_fails_on_invalid_parameters(self): hash_dim=-5, block_shape=(8, 8), algorithm='gradient') len(result) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_gradient_can_use_block(self): config = { "label1": 100, @@ -256,6 +266,7 @@ def test_ndr_gradient_can_use_block(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_gradient_can_use_hash_dim(self): config = { "label1": 100, @@ -278,6 +289,7 @@ def test_ndr_gradient_can_use_hash_dim(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_gradient_can_use_sim_thresh(self): config = { "label1": 100, @@ -300,6 +312,7 @@ def test_ndr_gradient_can_use_sim_thresh(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_ndr_seed(self): config = { "label1": 100, diff --git a/tests/test_ops.py b/tests/test_ops.py index 7294614a20..8658015dc8 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -10,9 +10,11 @@ compute_ann_statistics, mean_std, find_unique_images) from datumaro.components.dataset import Dataset from datumaro.util.test_utils import compare_datasets +from .requirements import Requirements, mark_requirement class TestOperations(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_mean_std(self): expected_mean = [100, 50, 150] expected_std = [20, 50, 10] @@ -33,6 +35,7 @@ def test_mean_std(self): for estd, astd in zip(expected_std, actual_std): self.assertAlmostEqual(estd, astd, places=0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_stats(self): dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.ones((5, 5, 3)), annotations=[ @@ -138,6 +141,7 @@ def test_stats(self): self.assertEqual(expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_stats_with_empty_dataset(self): dataset = Dataset.from_iterable([ DatasetItem(id=1), @@ -189,6 +193,7 @@ def test_stats_with_empty_dataset(self): self.assertEqual(expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_unique_image_count(self): expected = { frozenset([('1', 'a'), ('1', 'b')]), @@ -215,6 +220,7 @@ def test_unique_image_count(self): class TestMultimerge(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_match_items(self): # items 1 and 3 are unique, item 2 is common and should be merged @@ -271,6 +277,7 @@ def test_can_match_items(self): key=lambda e: e.item_id) ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_match_shapes(self): source0 = Dataset.from_iterable([ DatasetItem(1, annotations=[ @@ -374,6 +381,7 @@ def test_can_match_shapes(self): key=lambda e: len(e.sources)) ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_attributes(self): source0 = Dataset.from_iterable([ DatasetItem(1, annotations=[ @@ -420,6 +428,7 @@ def test_attributes(self): if isinstance(e, FailedAttrVotingError)]) ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_group_checks(self): dataset = Dataset.from_iterable([ DatasetItem(1, annotations=[ @@ -446,6 +455,7 @@ def test_group_checks(self): if isinstance(e, WrongGroupError)]), merger.errors ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_merge_classes(self): source0 = Dataset.from_iterable([ DatasetItem(1, annotations=[ @@ -479,6 +489,7 @@ def test_can_merge_classes(self): compare_datasets(self, expected, merged, ignored_attrs={'score'}) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_merge_categories(self): source0 = Dataset.from_iterable([ DatasetItem(1, annotations=[ Label(0), ]), diff --git a/tests/test_project.py b/tests/test_project.py index b4ab7bbf58..8a8ddbaccf 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -12,9 +12,11 @@ from datumaro.components.config import Config from datumaro.components.dataset import Dataset, DEFAULT_FORMAT from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class ProjectTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_generate(self): src_config = Config({ 'project_name': 'test_project', @@ -34,13 +36,16 @@ def test_project_generate(self): src_config.format_version, result_config.format_version) @staticmethod + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_default_ctor_is_ok(): Project() @staticmethod + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_empty_config_is_ok(): Project(Config()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_add_source(self): source_name = 'source' origin = Source({ @@ -55,6 +60,7 @@ def test_add_source(self): self.assertIsNotNone(added) self.assertEqual(added, origin) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_added_source_can_be_saved(self): source_name = 'source' origin = Source({ @@ -67,6 +73,7 @@ def test_added_source_can_be_saved(self): self.assertEqual(origin, saved.sources[source_name]) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_added_source_can_be_dumped(self): source_name = 'source' origin = Source({ @@ -82,6 +89,7 @@ def test_added_source_can_be_dumped(self): loaded = loaded.get_source(source_name) self.assertEqual(origin, loaded) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import_with_custom_importer(self): class TestImporter: def __call__(self, path, subset=None): @@ -102,6 +110,7 @@ def __call__(self, path, subset=None): self.assertEqual(path, project.config.project_filename) self.assertListEqual(['train'], project.config.subsets) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_dump_added_model(self): model_name = 'model' @@ -116,6 +125,7 @@ def test_can_dump_added_model(self): loaded = loaded.get_model(model_name) self.assertEqual(saved, loaded) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_have_project_source(self): with TestDir() as test_dir: Project.generate(test_dir) @@ -128,6 +138,7 @@ def test_can_have_project_source(self): self.assertTrue('project1' in dataset.sources) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_batch_launch_custom_model(self): dataset = Dataset.from_iterable([ DatasetItem(id=i, subset='train', image=np.array([i])) @@ -157,6 +168,7 @@ def launch(self, inputs): self.assertEqual(int(item.id), item.annotations[0].attributes['data']) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_do_transform_with_custom_model(self): class TestExtractorSrc(Extractor): def __iter__(self): @@ -209,6 +221,7 @@ def __iter__(self): self.assertEqual(0, item1.annotations[0].label) self.assertEqual(1, item2.annotations[0].label) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_source_datasets_can_be_merged(self): class TestExtractor(Extractor): def __init__(self, url, n=0, s=0): @@ -235,6 +248,7 @@ def __iter__(self): self.assertEqual(n1 + n2, len(dataset)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_cant_merge_different_categories(self): class TestExtractor1(Extractor): def __iter__(self): @@ -264,6 +278,7 @@ def categories(self): with self.assertRaisesRegex(Exception, "different categories"): project.make_dataset() + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_filter_can_be_applied(self): class TestExtractor(Extractor): def __iter__(self): @@ -279,6 +294,7 @@ def __iter__(self): self.assertEqual(5, len(dataset)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_own_dataset(self): with TestDir() as test_dir: src_project = Project() @@ -292,6 +308,7 @@ def test_can_save_and_load_own_dataset(self): self.assertEqual(list(src_dataset), list(loaded_dataset)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_own_dataset_can_be_modified(self): project = Project() dataset = project.make_dataset() @@ -301,6 +318,7 @@ def test_project_own_dataset_can_be_modified(self): self.assertEqual(item, next(iter(dataset))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_compound_child_can_be_modified_recursively(self): with TestDir() as test_dir: child1 = Project({ @@ -331,6 +349,7 @@ def test_project_compound_child_can_be_modified_recursively(self): self.assertEqual(1, len(dataset.sources['child1'])) self.assertEqual(1, len(dataset.sources['child2'])) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_project_can_merge_item_annotations(self): class TestExtractor1(Extractor): def __iter__(self): @@ -359,6 +378,7 @@ def __iter__(self): item = next(iter(merged)) self.assertEqual(3, len(item.annotations)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_and_import(self): env = Environment() env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} @@ -378,6 +398,7 @@ def test_can_detect_and_import(self): DEFAULT_FORMAT) compare_datasets(self, source_dataset, imported_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_custom_extractor_can_be_created(self): class CustomExtractor(Extractor): def __iter__(self): diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 3f3d316d28..4d4c10a9c0 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -19,6 +19,7 @@ has_libs = True except ImportError: has_libs = False +from .requirements import Requirements, mark_requirement @skipIf(not has_libs, "pandas library is not available") @@ -83,6 +84,7 @@ def _generate_classification_dataset(self, config, subset=None, dataset = Dataset.from_iterable(iterable, categories) return dataset + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_get_sample_classification(self): config = { "label1": 10, @@ -209,6 +211,7 @@ def test_sampler_get_sample_classification(self): num_pre_train_subset - len(result.get_subset("sample")), ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_gives_error(self): config = { "label1": 10, @@ -378,6 +381,7 @@ def test_sampler_gives_error(self): entropy(data_df, infer_df) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_get_invalid_data(self): with self.subTest("empty dataset"): config = { @@ -491,6 +495,7 @@ def test_sampler_get_invalid_data(self): result = iter(result) next(result) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_number_of_samples(self): config = { "label1": 10, @@ -675,6 +680,7 @@ def test_sampler_number_of_samples(self): ) self.assertEqual(len(result.get_subset("sample")), 9) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_accumulated_sampling(self): config = { "label1": 10, @@ -882,6 +888,7 @@ def test_sampler_accumulated_sampling(self): self.assertEqual(len(result.get_subset("sample")), 9) self.assertEqual(len(result.get_subset("test")), num_pre_test_subset - 4) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_unaccumulated_sampling(self): config = { "label1": 10, @@ -1095,6 +1102,7 @@ def test_sampler_unaccumulated_sampling(self): self.assertEqual(len(result.get_subset("sample3")), 4) self.assertEqual(len(result.get_subset("test")), num_pre_test_subset - 4) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_sampler_parser(self): from argparse import ArgumentParser diff --git a/tests/test_splitter.py b/tests/test_splitter.py index a6b778e192..21aa6ceb4d 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -15,6 +15,7 @@ import datumaro.plugins.splitter as splitter from datumaro.components.operations import compute_ann_statistics +from .requirements import Requirements, mark_requirement class SplitterTest(TestCase): @@ -69,6 +70,7 @@ def _generate_dataset(self, config): dataset = Dataset.from_iterable(iterable, categories) return dataset + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_multi_class_no_attr(self): config = { "label1": {"attrs": None, "counts": 10}, @@ -98,6 +100,7 @@ def test_split_for_classification_multi_class_no_attr(self): self.assertEqual(6, dist_test["label2"][0]) self.assertEqual(9, dist_test["label3"][0]) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_single_class_single_attr(self): counts = {0: 10, 1: 20, 2: 30} config = {"label": {"attrs": ["attr"], "counts": counts}} @@ -124,6 +127,7 @@ def test_split_for_classification_single_class_single_attr(self): self.assertEqual(6, attr_test["attr"]["distribution"]["1"][0]) self.assertEqual(9, attr_test["attr"]["distribution"]["2"][0]) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_single_class_multi_attr(self): counts = { (0, 0): 20, @@ -170,6 +174,7 @@ def test_split_for_classification_single_class_multi_attr(self): self.assertEqual(114, len(actual.get_subset("train"))) self.assertEqual(6, len(actual.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_multi_label_with_attr(self): counts = { (0, 0): 20, @@ -235,6 +240,7 @@ def test_split_for_classification_multi_label_with_attr(self): list(r1.get_subset("test")), list(r3.get_subset("test")) ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_zero_ratio(self): config = { "label1": {"attrs": None, "counts": 5}, @@ -249,6 +255,7 @@ def test_split_for_classification_zero_ratio(self): self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(0, len(actual.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_unlabeled(self): with self.subTest("no label"): iterable = [DatasetItem(i, annotations=[]) for i in range(10)] @@ -271,6 +278,7 @@ def test_split_for_classification_unlabeled(self): self.assertEqual(7, len(actual.get_subset("train"))) self.assertEqual(3, len(actual.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_classification_gives_error(self): source = Dataset.from_iterable( [ @@ -295,6 +303,7 @@ def test_split_for_classification_gives_error(self): splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] splitter.Split(source, task, splits) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_reidentification(self): """ Test ReidentificationSplit using Dataset with label (ImageNet style) @@ -372,6 +381,7 @@ def _get_present(stat): self.assertEqual(int(total * 0.3 / 0.7), dist_gallery[pid][0]) self.assertEqual(int(total * 0.4 / 0.7), dist_query[pid][0]) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_reidentification_randomseed(self): """ Test randomseed for reidentification @@ -393,6 +403,7 @@ def test_split_for_reidentification_randomseed(self): self.assertEqual(list(r1.get_subset("train")), list(r2.get_subset("train"))) self.assertNotEqual(list(r1.get_subset("train")), list(r3.get_subset("train"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_reidentification_rebalance(self): """ rebalance function shouldn't gives error when there's no exchange @@ -412,6 +423,7 @@ def test_split_for_reidentification_rebalance(self): self.assertEqual(90, len(actual.get_subset("test-gallery"))) self.assertEqual(120, len(actual.get_subset("test-query"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_reidentification_unlabeled(self): query = 0.5 task = splitter.SplitTask.reid.name @@ -432,6 +444,7 @@ def test_split_for_reidentification_unlabeled(self): self.assertEqual(10, len(actual.get_subset("not-supported"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_reidentification_gives_error(self): query = 0.4 / 0.7 # valid query ratio task = splitter.SplitTask.reid.name @@ -797,6 +810,7 @@ def append_polygon_labelme(annotations, **kwargs): func = functions.get(dataset_type, append_polygon_coco) return func + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_detection(self): dtypes = ["coco", "voc", "yolo", "cvat", "labelme", "mot", "widerface"] task = splitter.SplitTask.detection.name @@ -847,6 +861,7 @@ def test_split_for_detection(self): self.assertEqual(list(r1.get_subset("test")), list(r2.get_subset("test"))) self.assertNotEqual(list(r1.get_subset("test")), list(r3.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_detection_with_unlabeled(self): source, _ = self._generate_detection_segmentation_dataset( annotation_type=self._get_append_bbox("cvat"), @@ -863,6 +878,7 @@ def test_split_for_detection_with_unlabeled(self): self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_detection_gives_error(self): source, _ = self._generate_detection_segmentation_dataset( annotation_type=self._get_append_bbox("cvat"), @@ -885,6 +901,7 @@ def test_split_for_detection_gives_error(self): splits = [("train", 0.5), ("train", 0.2), ("test", 0.3)] splitter.Split(source, task, splits) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_no_subset_name_and_count_restriction(self): splits = [ ("_train", 0.5), @@ -945,6 +962,7 @@ def test_no_subset_name_and_count_restriction(self): self.assertEqual(2, len(actual.get_subset("test*"))) self.assertEqual(1, len(actual.get_subset("test2"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_segmentation(self): with self.subTest("mask annotation"): @@ -1054,6 +1072,7 @@ def test_split_for_segmentation(self): list(r1.get_subset("test")), list(r3.get_subset("test")) ) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_segmentation_with_unlabeled(self): with self.subTest("mask annotation"): @@ -1088,6 +1107,7 @@ def test_split_for_segmentation_with_unlabeled(self): self.assertEqual(4, len(actual.get_subset("val"))) self.assertEqual(6, len(actual.get_subset("test"))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_split_for_segmentation_gives_error(self): with self.subTest("mask annotation"): diff --git a/tests/test_tfrecord_format.py b/tests/test_tfrecord_format.py index 6db7c07eb3..1b147fa3e8 100644 --- a/tests/test_tfrecord_format.py +++ b/tests/test_tfrecord_format.py @@ -13,6 +13,7 @@ from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) from datumaro.util.tf_util import check_import +from .requirements import Requirements, mark_requirement try: from datumaro.plugins.tf_detection_api_format.extractor import \ @@ -28,6 +29,7 @@ @skipIf(not module_found, "Tensorflow package is not found") class TfImportTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_raises_when_crashes_on_import(self): # Should fire if import can't be done for any reason except # module unavailability and import crash @@ -42,6 +44,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='tf_detection_api', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_bboxes(self): test_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -63,6 +66,7 @@ def test_can_save_bboxes(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_masks(self): test_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((4, 5, 3)), @@ -87,6 +91,7 @@ def test_can_save_masks(self): partial(TfDetectionApiConverter.convert, save_masks=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): test_dataset = Dataset.from_iterable([ DatasetItem(id=1, @@ -121,6 +126,7 @@ def test_can_save_dataset_with_no_subsets(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): test_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', @@ -142,6 +148,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_image_info(self): test_dataset = Dataset.from_iterable([ DatasetItem(id='1/q.e', @@ -154,6 +161,7 @@ def test_can_save_dataset_with_image_info(self): self._test_save_and_load(test_dataset, TfDetectionApiConverter.convert, test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_unknown_image_formats(self): test_dataset = Dataset.from_iterable([ DatasetItem(id=1, @@ -173,6 +181,7 @@ def test_can_save_dataset_with_unknown_image_formats(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem('q/1', subset='train', @@ -188,6 +197,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset @@ -209,6 +219,7 @@ def test_inplace_save_writes_only_updated_data(self): self.assertFalse(osp.isfile(osp.join(path, 'b.tfrecord'))) self.assertTrue(osp.isfile(osp.join(path, 'c.tfrecord'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_labelmap_parsing(self): text = """ { @@ -241,9 +252,11 @@ def test_labelmap_parsing(self): @skipIf(import_failed, "Failed to import tensorflow") class TfrecordImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(TfDetectionApiImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): target_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 1e310c1832..9ccd45023e 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -10,9 +10,11 @@ import datumaro.util.mask_tools as mask_tools import datumaro.plugins.transforms as transforms from datumaro.util.test_utils import compare_datasets +from .requirements import Requirements, mark_requirement class TransformsTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_reindex(self): source = Dataset.from_iterable([ DatasetItem(id=10), @@ -29,6 +31,7 @@ def test_reindex(self): actual = transforms.Reindex(source, start=5) compare_datasets(self, expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_mask_to_polygons(self): source = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 10, 3)), annotations=[ @@ -53,6 +56,7 @@ def test_mask_to_polygons(self): actual = transforms.MasksToPolygons(source) compare_datasets(self, expected, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_mask_to_polygons_small_polygons_message(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 10, 3)), annotations=[ @@ -74,6 +78,7 @@ def test_mask_to_polygons_small_polygons_message(self): compare_datasets(self, target_dataset, actual) self.assertRegex('\n'.join(logs.output), 'too small polygons') + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_polygons_to_masks(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 10, 3)), annotations=[ @@ -106,6 +111,7 @@ def test_polygons_to_masks(self): actual = transforms.PolygonsToMasks(source_dataset) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_crop_covered_segments(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 5, 3)), annotations=[ @@ -137,6 +143,7 @@ def test_crop_covered_segments(self): actual = transforms.CropCoveredSegments(source_dataset) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_merge_instance_segments(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 5, 3)), @@ -184,6 +191,7 @@ def test_merge_instance_segments(self): include_polygons=True) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_map_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='a'), @@ -201,6 +209,7 @@ def test_map_subsets(self): { 'a': '', 'b': 'a' }) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_shapes_to_boxes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 5, 3)), @@ -233,6 +242,7 @@ def test_shapes_to_boxes(self): actual = transforms.ShapesToBoxes(source_dataset) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_id_from_image(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image='path.jpg'), @@ -246,6 +256,7 @@ def test_id_from_image(self): actual = transforms.IdFromImageName(source_dataset) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_boxes_to_masks(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, image=np.zeros((5, 5, 3)), @@ -291,6 +302,7 @@ def test_boxes_to_masks(self): actual = transforms.BoxesToMasks(source_dataset) compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_random_split(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset="a"), @@ -310,6 +322,7 @@ def test_random_split(self): self.assertEqual(4, len(actual.get_subset('train'))) self.assertEqual(3, len(actual.get_subset('test'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_random_split_gives_error_on_wrong_ratios(self): source_dataset = Dataset.from_iterable([DatasetItem(id=1)]) @@ -328,6 +341,7 @@ def test_random_split_gives_error_on_wrong_ratios(self): ('test', 1.5), ]) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_remap_labels(self): src_dataset = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ @@ -378,6 +392,7 @@ def test_remap_labels(self): compare_datasets(self, dst_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_remap_labels_delete_unspecified(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ @@ -398,6 +413,7 @@ def test_remap_labels_delete_unspecified(self): compare_datasets(self, target_dataset, actual) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_transform_labels(self): src_dataset = Dataset.from_iterable([ DatasetItem(id=1, annotations=[ diff --git a/tests/test_util.py b/tests/test_util.py index f19e5d4f95..88f850eb3a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -6,9 +6,11 @@ from datumaro.util import Rollback, error_rollback from datumaro.util.test_utils import TestDir from datumaro.util.os_util import walk +from .requirements import Requirements, mark_requirement class TestRollback(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_does_not_call_on_no_error(self): success = True def cb(): @@ -20,6 +22,7 @@ def cb(): self.assertTrue(success) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_calls_on_error(self): success = False def cb(): @@ -35,6 +38,7 @@ def cb(): finally: self.assertTrue(success) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_decorator_calls_on_error(self): success = False def cb(): @@ -53,6 +57,7 @@ def foo(on_error=None): finally: self.assertTrue(success) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_decorator_does_not_call_on_no_error(self): success = True def cb(): @@ -67,6 +72,7 @@ def foo(on_error=None): self.assertTrue(success) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_decorator_supports_implicit_arg(self): success = False def cb(): @@ -85,6 +91,7 @@ def foo(): finally: self.assertTrue(success) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_fowrard_args(self): success1 = False def cb1(a1, a2=None, ignore_errors=None): @@ -111,6 +118,7 @@ def cb2(a1, a2=None, ignore_errors=None): self.assertTrue(success2) class TestOsUtils(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_walk_with_maxdepth(self): with TestDir() as rootdir: os.makedirs(osp.join(rootdir, '1', '2', '3', '4')) diff --git a/tests/test_validator.py b/tests/test_validator.py index 041e7e66b9..2d0bd47e78 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -19,6 +19,7 @@ from datumaro.components.validator import (ClassificationValidator, DetectionValidator, TaskType, validate_annotations, _Validator, SegmentationValidator) +from .requirements import Requirements, mark_requirement class TestValidatorTemplate(TestCase): @@ -117,10 +118,12 @@ def setUpClass(cls): few_samples_thr=1, imbalance_ratio_thr=50, far_from_mean_thr=5.0, dominance_ratio_thr=0.8, topk_bins=0.1) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_generate_reports(self): with self.assertRaises(NotImplementedError): self.validator.generate_reports({}) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_missing_label_categories(self): stats = { 'label_distribution': { @@ -133,6 +136,7 @@ def test_check_missing_label_categories(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], MissingLabelCategories) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_missing_attribute(self): label_name = 'unit' attr_name = 'test' @@ -146,6 +150,7 @@ def test_check_missing_attribute(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], MissingAttribute) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_undefined_label(self): label_name = 'unittest' label_stats = { @@ -158,6 +163,7 @@ def test_check_undefined_label(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], UndefinedLabel) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_undefined_attribute(self): label_name = 'unit' attr_name = 'test' @@ -171,6 +177,7 @@ def test_check_undefined_attribute(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], UndefinedAttribute) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_label_defined_but_not_found(self): stats = { 'label_distribution': { @@ -186,6 +193,7 @@ def test_check_label_defined_but_not_found(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], LabelDefinedButNotFound) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_attribute_defined_but_not_found(self): label_name = 'unit' attr_stats = { @@ -200,6 +208,7 @@ def test_check_attribute_defined_but_not_found(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], AttributeDefinedButNotFound) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_only_one_label(self): stats = { 'label_distribution': { @@ -215,6 +224,7 @@ def test_check_only_one_label(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], OnlyOneLabel) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_only_one_attribute_value(self): label_name = 'unit' attr_name = 'test' @@ -230,6 +240,7 @@ def test_check_only_one_attribute_value(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], OnlyOneAttributeValue) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_few_samples_in_label(self): with self.subTest('Few Samples'): stats = { @@ -258,6 +269,7 @@ def test_check_few_samples_in_label(self): self.assertTrue(len(actual_reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_few_samples_in_attribute(self): label_name = 'unit' attr_name = 'test' @@ -287,6 +299,7 @@ def test_check_few_samples_in_attribute(self): self.assertTrue(len(actual_reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_labels(self): with self.subTest('Imbalance'): stats = { @@ -317,6 +330,7 @@ def test_check_imbalanced_labels(self): self.assertTrue(len(actual_reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_attribute(self): label_name = 'unit' attr_name = 'test' @@ -356,6 +370,7 @@ def setUpClass(cls): imbalance_ratio_thr=50, far_from_mean_thr=5.0, dominance_ratio_thr=0.8, topk_bins=0.1) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_missing_label_annotation(self): stats = { 'items_missing_annotation': [(1, 'unittest')] @@ -366,6 +381,7 @@ def test_check_missing_label_annotation(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], MissingAnnotation) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_multi_label_annotations(self): stats = { 'items_with_multiple_labels': [(1, 'unittest')] @@ -384,6 +400,7 @@ def setUpClass(cls): imbalance_ratio_thr=50, far_from_mean_thr=5.0, dominance_ratio_thr=0.8, topk_bins=0.1) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_dist_in_label(self): label_name = 'unittest' most = int(self.validator.dominance_thr * 100) @@ -416,6 +433,7 @@ def test_check_imbalanced_dist_in_label(self): self.assertTrue(len(reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_dist_in_attr(self): label_name = 'unit' attr_name = 'test' @@ -455,6 +473,7 @@ def test_check_imbalanced_dist_in_attr(self): self.assertTrue(len(reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_missing_bbox_annotation(self): stats = { 'items_missing_annotation': [(1, 'unittest')] @@ -465,6 +484,7 @@ def test_check_missing_bbox_annotation(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], MissingAnnotation) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_negative_length(self): stats = { 'items_with_negative_length': { @@ -481,6 +501,7 @@ def test_check_negative_length(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], NegativeLength) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_invalid_value(self): stats = { 'items_with_invalid_value': { @@ -495,6 +516,7 @@ def test_check_invalid_value(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], InvalidValue) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_far_from_label_mean(self): label_name = 'unittest' bbox_label_stats = { @@ -514,6 +536,7 @@ def test_check_far_from_label_mean(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], FarFromLabelMean) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_far_from_attr_mean(self): label_name = 'unit' attr_name = 'test' @@ -544,6 +567,7 @@ def setUpClass(cls): imbalance_ratio_thr=50, far_from_mean_thr=5.0, dominance_ratio_thr=0.8, topk_bins=0.1) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_dist_in_label(self): label_name = 'unittest' most = int(self.validator.dominance_thr * 100) @@ -576,6 +600,7 @@ def test_check_imbalanced_dist_in_label(self): self.assertTrue(len(reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_imbalanced_dist_in_attr(self): label_name = 'unit' attr_name = 'test' @@ -615,6 +640,7 @@ def test_check_imbalanced_dist_in_attr(self): self.assertTrue(len(reports) == 0) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_missing_mask_annotation(self): stats = { 'items_missing_annotation': [(1, 'unittest')] @@ -625,6 +651,7 @@ def test_check_missing_mask_annotation(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], MissingAnnotation) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_invalid_value(self): stats = { 'items_with_invalid_value': { @@ -639,6 +666,7 @@ def test_check_invalid_value(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], InvalidValue) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_far_from_label_mean(self): label_name = 'unittest' mask_label_stats = { @@ -658,6 +686,7 @@ def test_check_far_from_label_mean(self): self.assertTrue(len(actual_reports) == 1) self.assertIsInstance(actual_reports[0], FarFromLabelMean) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_check_far_from_attr_mean(self): label_name = 'unit' attr_name = 'test' @@ -682,6 +711,7 @@ def test_check_far_from_attr_mean(self): class TestValidateAnnotations(TestValidatorTemplate): + extra_args = { 'few_samples_thr': 1, 'imbalance_ratio_thr': 50, @@ -689,6 +719,7 @@ class TestValidateAnnotations(TestValidatorTemplate): 'dominance_ratio_thr': 0.8, 'topk_bins': 0.1, } + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_validate_annotations_classification(self): actual_results = validate_annotations(self.dataset, 'classification', **self.extra_args) @@ -745,6 +776,7 @@ def test_validate_annotations_classification(self): self.assertEqual(actual_summary, expected_summary) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_validate_annotations_detection(self): actual_results = validate_annotations(self.dataset, 'detection', **self.extra_args) @@ -799,6 +831,7 @@ def test_validate_annotations_detection(self): self.assertEqual(actual_summary, expected_summary) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_validate_annotations_segmentation(self): actual_results = validate_annotations(self.dataset, 'segmentation', **self.extra_args) @@ -854,10 +887,12 @@ def test_validate_annotations_segmentation(self): self.assertEqual(actual_summary, expected_summary) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_validate_annotations_invalid_task_type(self): with self.assertRaises(ValueError): validate_annotations(self.dataset, 'INVALID', **self.extra_args) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_validate_annotations_invalid_dataset_type(self): with self.assertRaises(TypeError): validate_annotations(object(), 'classification', **self.extra_args) diff --git a/tests/test_vgg_face2_format.py b/tests/test_vgg_face2_format.py index d6d232a921..9801f44d17 100644 --- a/tests/test_vgg_face2_format.py +++ b/tests/test_vgg_face2_format.py @@ -9,9 +9,11 @@ VggFace2Importer) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class VggFace2FormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', image=np.ones((8, 8, 3)), @@ -55,6 +57,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='b/1', image=np.ones((8, 8, 3)), @@ -72,6 +75,7 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', image=np.ones((8, 8, 3)), @@ -89,6 +93,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((8, 8, 3)), @@ -106,6 +111,7 @@ def test_can_save_dataset_with_no_save_images(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_labels(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((8, 8, 3)), @@ -128,6 +134,7 @@ def test_can_save_dataset_with_no_labels(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_wrong_number_of_points(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((8, 8, 3)), @@ -149,6 +156,7 @@ def test_can_save_dataset_with_wrong_number_of_points(self): compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem('q/1', image=Image(path='q/1.JPEG', @@ -172,9 +180,11 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'vgg_face2_dataset') class VggFace2ImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(VggFace2Importer.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='0001_01', subset='train', diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index 5f30ccd1f9..f8b7df6719 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -24,9 +24,11 @@ from datumaro.util.mask_tools import load_mask from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) +from .requirements import Requirements, mark_requirement class VocFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_colormap_generator(self): reference = np.array([ [ 0, 0, 0], @@ -55,6 +57,7 @@ def test_colormap_generator(self): self.assertTrue(np.array_equal(reference, list(VOC.VocColormap.values()))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_write_and_parse_labelmap(self): src_label_map = VOC.make_voc_label_map() src_label_map['qq'] = [None, ['part1', 'part2'], ['act1', 'act2']] @@ -79,6 +82,7 @@ def categories(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'voc_dataset', 'voc_dataset1') class VocImportTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): class DstExtractor(TestExtractorBase): def __iter__(self): @@ -128,6 +132,7 @@ def __iter__(self): compare_datasets(self, DstExtractor(), dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_voc(self): self.assertTrue(VocImporter.detect(DUMMY_DATASET_DIR)) @@ -138,6 +143,7 @@ def _test_save_and_load(self, source_dataset, converter, test_dir, importer='voc', target_dataset=target_dataset, importer_args=importer_args, **kwargs) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_cls(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -158,6 +164,7 @@ def __iter__(self): partial(VocClassificationConverter.convert, label_map='voc'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_det(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -214,6 +221,7 @@ def __iter__(self): partial(VocDetectionConverter.convert, label_map='voc'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_segm(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -248,6 +256,7 @@ def __iter__(self): partial(VocSegmentationConverter.convert, label_map='voc'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_segm_unpainted(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -283,6 +292,7 @@ def __iter__(self): label_map='voc', apply_colormap=False), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_segm_with_many_instances(self): def bit(x, y, shape): mask = np.zeros(shape) @@ -318,6 +328,7 @@ def __iter__(self): partial(VocSegmentationConverter.convert, label_map='voc'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_layout(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -342,6 +353,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(VocLayoutConverter.convert, label_map='voc'), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_voc_action(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -398,6 +410,7 @@ def __iter__(self): label_map='voc', allow_attributes=False), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -412,6 +425,7 @@ def __iter__(self): partial(VocConverter.convert, label_map='voc', tasks=task), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -428,6 +442,7 @@ def __iter__(self): save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_images(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -445,6 +460,7 @@ def __iter__(self): save_images=True, tasks=task), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_voc_labelmap(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -482,6 +498,7 @@ def categories(self): partial(VocConverter.convert, label_map='voc'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_source_labelmap_undefined(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -529,6 +546,7 @@ def categories(self): partial(VocConverter.convert, label_map='source'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_source_labelmap_defined(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -575,6 +593,7 @@ def categories(self): partial(VocConverter.convert, label_map='source'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_dataset_with_fixed_labelmap(self): class SrcExtractor(TestExtractorBase): def __iter__(self): @@ -629,6 +648,7 @@ def categories(self): partial(VocConverter.convert, label_map=label_map), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_background_masks_dont_introduce_instances_but_cover_others(self): dataset = Dataset.from_iterable([ DatasetItem(1, image=np.zeros((4, 1, 1)), annotations=[ @@ -648,6 +668,7 @@ def test_background_masks_dont_introduce_instances_but_cover_others(self): self.assertTrue(np.array_equal([0, 1], np.unique(cls_mask))) self.assertTrue(np.array_equal([0, 1], np.unique(inst_mask))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_image_info(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -661,6 +682,7 @@ def __iter__(self): partial(VocConverter.convert, label_map='voc', tasks=task), test_dir) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -678,6 +700,7 @@ def __iter__(self): save_images=True), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -694,6 +717,7 @@ def __iter__(self): label_map='voc', save_images=True, tasks=task), test_dir, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_attributes(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -725,6 +749,7 @@ def __iter__(self): partial(VocConverter.convert, label_map='voc'), test_dir, target_dataset=DstExtractor()) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset @@ -765,6 +790,7 @@ def test_inplace_save_writes_only_updated_data(self): self.assertFalse(osp.isfile( osp.join(path, 'SegmentationClass', '3.png'))) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_data_images(self): class TestExtractor(TestExtractorBase): def __iter__(self): diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index 4d733ee82a..a6b4ab3ccf 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -8,9 +8,11 @@ from datumaro.plugins.widerface_format import WiderFaceConverter, WiderFaceImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class WiderFaceFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', image=np.ones((8, 8, 3)), @@ -62,6 +64,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), @@ -80,6 +83,7 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', image=np.ones((8, 8, 3)), @@ -98,6 +102,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_non_widerface_attributes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), @@ -129,6 +134,7 @@ def test_can_save_dataset_with_non_widerface_attributes(self): compare_datasets(self, target_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem('q/1', image=Image(path='q/1.JPEG', @@ -146,9 +152,11 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'widerface_dataset') class WiderFaceImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(WiderFaceImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='0_Parade_image_01', subset='train', diff --git a/tests/test_yolo_format.py b/tests/test_yolo_format.py index 5449ba6626..a537e8d736 100644 --- a/tests/test_yolo_format.py +++ b/tests/test_yolo_format.py @@ -12,9 +12,11 @@ from datumaro.plugins.yolo_format.converter import YoloConverter from datumaro.util.image import Image, save_image from datumaro.util.test_utils import TestDir, compare_datasets +from .requirements import Requirements, mark_requirement class YoloFormatTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', image=np.ones((8, 8, 3)), @@ -47,6 +49,7 @@ def test_can_save_and_load(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_image_info(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -69,6 +72,7 @@ def test_can_save_dataset_with_image_info(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_load_dataset_with_exact_image_info(self): source_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -90,6 +94,7 @@ def test_can_load_dataset_with_exact_image_info(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='кириллица с пробелом', subset='train', image=np.ones((8, 8, 3)), @@ -109,6 +114,7 @@ def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_relative_paths(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -128,6 +134,7 @@ def test_relative_paths(self): compare_datasets(self, source_dataset, parsed_dataset) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_save_and_load_image_with_arbitrary_extension(self): dataset = Dataset.from_iterable([ DatasetItem('q/1', subset='train', @@ -142,6 +149,7 @@ def test_can_save_and_load_image_with_arbitrary_extension(self): compare_datasets(self, dataset, parsed_dataset, require_images=True) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset @@ -171,9 +179,11 @@ def test_inplace_save_writes_only_updated_data(self): DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'yolo_dataset') class YoloImporterTest(TestCase): + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): self.assertTrue(YoloImporter.detect(DUMMY_DATASET_DIR)) + @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', From 46d95162ed0480b4fbfd80fcc4264e8e9f7f7d29 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 2 Jun 2021 17:00:04 +0300 Subject: [PATCH 36/40] Fix CI (#272) * Fix script call * change script call to binary call --- .github/workflows/health_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/health_check.yml b/.github/workflows/health_check.yml index e721a24d4e..611e6c0a17 100644 --- a/.github/workflows/health_check.yml +++ b/.github/workflows/health_check.yml @@ -23,7 +23,7 @@ jobs: - name: Code instrumentation run: | pytest -v --cov --cov-report xml:coverage.xml - datum.py -h + datum -h - name: Sending coverage results if: matrix.python-version == '3.6' run: | From e92da58d56c3be2cc8e1c6b3869cdb65a400d52c Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 3 Jun 2021 18:20:50 +0300 Subject: [PATCH 37/40] Fix help program name, add mark_bug (#275) * Fix prog name * Add mark_bug test annotation --- datumaro/cli/__main__.py | 5 ++++- tests/requirements.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datumaro/cli/__main__.py b/datumaro/cli/__main__.py index 2ecf9f7a78..528c6d019e 100644 --- a/datumaro/cli/__main__.py +++ b/datumaro/cli/__main__.py @@ -5,6 +5,7 @@ import argparse import logging as log +import os.path as osp import sys from . import contexts, commands @@ -50,9 +51,11 @@ def _make_subcommands_help(commands, help_line_start=0): return desc def make_parser(): - parser = argparse.ArgumentParser(prog="datumaro", + parser = argparse.ArgumentParser( description="Dataset Framework", formatter_class=argparse.RawDescriptionHelpFormatter) + if parser.prog == osp.basename(__file__): # python -m datumaro ... + parser.prog = 'datumaro' parser.add_argument('--version', action='version', version=VERSION) _LogManager._define_loglevel_option(parser) diff --git a/tests/requirements.py b/tests/requirements.py index 9bc8c1ad45..49ab421d62 100644 --- a/tests/requirements.py +++ b/tests/requirements.py @@ -16,6 +16,17 @@ def test_wrapper(*args, **kwargs): return test_wrapper return wrapper +def mark_bug(bugs): + def wrapper(test_func): + @pytest.mark.components(DatumaroComponent.Datumaro) + @pytest.mark.component + @pytest.mark.priority_medium + @pytest.mark.bugs(bugs) + def test_wrapper(*args, **kwargs): + return test_func(*args, **kwargs) + return test_wrapper + return wrapper + class DatumaroComponent: Datumaro = "datumaro" From 23a92b81d97082de7338f4c84ed8ecd7e826a144 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 3 Jun 2021 18:21:27 +0300 Subject: [PATCH 38/40] Fix labelmap parameter in CamVid (#262) * Fix labelmap parameter in camvid --- datumaro/plugins/camvid_format.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/datumaro/plugins/camvid_format.py b/datumaro/plugins/camvid_format.py index 76de818196..0abca1751b 100644 --- a/datumaro/plugins/camvid_format.py +++ b/datumaro/plugins/camvid_format.py @@ -218,6 +218,16 @@ def find_sources(cls, path): class CamvidConverter(Converter): DEFAULT_IMAGE_EXT = CamvidPath.IMAGE_EXT + @staticmethod + def _get_labelmap(s): + if osp.isfile(s): + return s + try: + return LabelmapType[s].name + except KeyError: + import argparse + raise argparse.ArgumentTypeError() + @classmethod def build_cmdline_parser(cls, **kwargs): parser = super().build_cmdline_parser(**kwargs) From bf32c4dc4091b8d99a72aeaa77784aae7d5cef6f Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 3 Jun 2021 18:28:18 +0300 Subject: [PATCH 39/40] Release 0.1.9 (dev) (#276) * Update version * Update changelog --- CHANGELOG.md | 4 ++-- datumaro/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 859379c6b4..94b8d7b49d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## 03/06/2021 - Release v0.1.9 ### Added - Support for escaping in attribute values in LabelMe format () - Support for Segmentation Splitting () @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Documentation file for COCO format () - Documentation file and integration tests for YOLO format () - Support for Cityscapes dataset format () -- Support for Validator configurable threshold() +- Support for Validator configurable threshold () ### Changed - LabelMe format saves dataset items with their relative paths by subsets without changing names () diff --git a/datumaro/version.py b/datumaro/version.py index 43ac631f58..aae69457aa 100644 --- a/datumaro/version.py +++ b/datumaro/version.py @@ -1 +1 @@ -VERSION = '0.1.8' \ No newline at end of file +VERSION = '0.1.9' \ No newline at end of file From 723c5004421ab830c37442e9b8fe7d24d2d660b7 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 3 Jun 2021 19:37:33 +0300 Subject: [PATCH 40/40] Fix numpy conflict (#278) --- requirements.txt | 2 +- setup.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c067c3b3e2..b5490d77d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ lxml>=4.4.1 matplotlib>=3.3.1 opencv-python-headless>=4.1.0.25 Pillow>=6.1.0 -pycocotools>=2.0.0 +pycocotools>=2.0.0 --no-binary=pycocotools # https://github.com/openvinotoolkit/datumaro/issues/253 PyYAML>=5.3.1 scikit-image>=0.15.0 tensorboardX>=1.8 diff --git a/setup.py b/setup.py index d1e5ff0152..68591d9242 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,17 @@ def get_requirements(): 'matplotlib', 'numpy>=1.17.3', 'Pillow', - 'pycocotools; platform_system != "Windows"', + + # Avoid 2.0.2 Linux binary distribution because of + # a conflict in numpy versions with TensorFlow: + # - TF is compiled with numpy 1.19 ABI + # - pycocotools is compiled with numpy 1.20 ABI + # Using a previous version allows to force package rebuilding. + # + # https://github.com/openvinotoolkit/datumaro/issues/253 + 'pycocotools!=2.0.2; platform_system != "Windows"', 'pycocotools-windows; platform_system == "Windows"', + 'PyYAML', 'scikit-image', 'tensorboardX',