From dd6130cbdd3133c6e5f2c6a3e4823ffa0e30f411 Mon Sep 17 00:00:00 2001 From: Galina Date: Fri, 18 Aug 2023 14:28:46 +0300 Subject: [PATCH 1/3] Fix label list for h-label cls --- .../classification/adapters/openvino/task.py | 18 +++++++++++++++--- src/otx/algorithms/classification/task.py | 14 +++++++++++--- src/otx/api/utils/dataset_utils.py | 12 ++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/otx/algorithms/classification/adapters/openvino/task.py b/src/otx/algorithms/classification/adapters/openvino/task.py index 741c9a6526a..0c07fe38603 100644 --- a/src/otx/algorithms/classification/adapters/openvino/task.py +++ b/src/otx/algorithms/classification/adapters/openvino/task.py @@ -76,7 +76,7 @@ IOptimizationTask, OptimizationType, ) -from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item +from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item, get_hierarchical_label_list logger = logging.getLogger(__name__) @@ -228,12 +228,18 @@ def add_prediction(id: int, predicted_scene: AnnotationSceneEntity, aux_data: tu if saliency_map is not None and repr_vector is not None: feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1)) dataset_item.append_metadata_item(feature_vec_media, model=self.model) + label_list = self.task_environment.get_labels() + # Fix the order for hierarchical labels to adjust classes with model outputs + if self.inferencer.model.hierarchical: + label_list = get_hierarchical_label_list( + self.inferencer.model.hierarchical_info["cls_heads_info"], label_list + ) add_saliency_maps_to_dataset_item( dataset_item=dataset_item, saliency_map=saliency_map, model=self.model, - labels=self.task_environment.get_labels(), + labels=label_list, predicted_scored_labels=item_labels, explain_predicted_classes=explain_predicted_classes, process_saliency_maps=process_saliency_maps, @@ -284,6 +290,12 @@ def explain( explain_predicted_classes = explain_parameters.explain_predicted_classes dataset_size = len(dataset) + label_list = self.task_environment.get_labels() + # Fix the order for hierarchical labels to adjust classes with model outputs + if self.inferencer.model.hierarchical: + label_list = get_hierarchical_label_list( + self.inferencer.model.hierarchical_info["cls_heads_info"], label_list + ) for i, dataset_item in enumerate(dataset, 1): predicted_scene, _, saliency_map, _, _ = self.inferencer.predict(dataset_item.numpy) if saliency_map is None: @@ -298,7 +310,7 @@ def explain( dataset_item=dataset_item, saliency_map=saliency_map, model=self.model, - labels=self.task_environment.get_labels(), + labels=label_list, predicted_scored_labels=item_labels, explain_predicted_classes=explain_predicted_classes, process_saliency_maps=process_saliency_maps, diff --git a/src/otx/algorithms/classification/task.py b/src/otx/algorithms/classification/task.py index 3c74230dcab..9faf4e931bf 100644 --- a/src/otx/algorithms/classification/task.py +++ b/src/otx/algorithms/classification/task.py @@ -75,7 +75,7 @@ from otx.api.serialization.label_mapper import label_schema_to_bytes from otx.api.usecases.evaluation.metrics_helper import MetricsHelper from otx.api.usecases.tasks.interfaces.export_interface import ExportType -from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item +from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item, get_hierarchical_label_list from otx.api.utils.labels_utils import get_empty_label from otx.cli.utils.multi_gpu import is_multigpu_child_process @@ -345,6 +345,10 @@ def _add_predictions_to_dataset( dataset_size = len(dataset) pos_thr = 0.5 + label_list = self._labels + # Fix the order for hierarchical labels to adjust classes with model outputs + if self._hierarchical: + label_list = get_hierarchical_label_list(self._hierarchical_info, label_list) for i, (dataset_item, prediction_items) in enumerate(zip(dataset, prediction_results)): prediction_item, feature_vector, saliency_map = prediction_items if any(np.isnan(prediction_item)): @@ -373,7 +377,7 @@ def _add_predictions_to_dataset( dataset_item=dataset_item, saliency_map=saliency_map, model=self._task_environment.model, - labels=self._labels, + labels=label_list, predicted_scored_labels=item_labels, explain_predicted_classes=explain_predicted_classes, process_saliency_maps=process_saliency_maps, @@ -436,13 +440,17 @@ def _add_explanations_to_dataset( ): """Loop over dataset again and assign saliency maps.""" dataset_size = len(dataset) + label_list = self._labels + # Fix the order for hierarchical labels to adjust classes with model outputs + if self._hierarchical: + label_list = get_hierarchical_label_list(self._hierarchical_info, label_list) for i, (dataset_item, prediction_item, saliency_map) in enumerate(zip(dataset, predictions, saliency_maps)): item_labels = self._get_item_labels(prediction_item, pos_thr=0.5) add_saliency_maps_to_dataset_item( dataset_item=dataset_item, saliency_map=saliency_map, model=self._task_environment.model, - labels=self._labels, + labels=label_list, predicted_scored_labels=item_labels, explain_predicted_classes=explain_predicted_classes, process_saliency_maps=process_saliency_maps, diff --git a/src/otx/api/utils/dataset_utils.py b/src/otx/api/utils/dataset_utils.py index 7f0cddd6a74..ad379a75b6a 100644 --- a/src/otx/api/utils/dataset_utils.py +++ b/src/otx/api/utils/dataset_utils.py @@ -272,3 +272,15 @@ def non_linear_normalization(saliency_map: np.ndarray) -> np.ndarray: saliency_map = 255.0 / (max_soft_score + 1e-12) * saliency_map return np.uint8(np.floor(saliency_map)) + + +def get_hierarchical_label_list(hierarchical_info, labels): + """Return hierarchical labels list which is adjusted to model outputs classes.""" + hierarchical_labels = [] + for head_idx in range(hierarchical_info["num_multiclass_heads"]): + logits_begin, logits_end = hierarchical_info["head_idx_to_logits_range"][str(head_idx)] + for logit in range(0, logits_end - logits_begin): + label_str = hierarchical_info["all_groups"][head_idx][logit] + otx_label = next(x for x in labels if x.name == label_str) + hierarchical_labels.append(otx_label) + return hierarchical_labels From 7ab3633c1bffe73c20968af8b014022ec75f57c1 Mon Sep 17 00:00:00 2001 From: Galina Date: Fri, 18 Aug 2023 16:01:34 +0300 Subject: [PATCH 2/3] Fixes from comments --- .../classification/adapters/openvino/task.py | 3 ++- src/otx/algorithms/classification/task.py | 3 ++- .../classification/utils/__init__.py | 2 ++ .../classification/utils/cls_utils.py | 21 +++++++++++++++++++ src/otx/api/utils/dataset_utils.py | 12 ----------- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/otx/algorithms/classification/adapters/openvino/task.py b/src/otx/algorithms/classification/adapters/openvino/task.py index 0c07fe38603..ed21e928af0 100644 --- a/src/otx/algorithms/classification/adapters/openvino/task.py +++ b/src/otx/algorithms/classification/adapters/openvino/task.py @@ -36,6 +36,7 @@ from otx.algorithms.classification.utils import ( get_cls_deploy_config, get_cls_inferencer_configuration, + get_hierarchical_label_list, ) from otx.algorithms.common.utils import OTXOpenVinoDataLoader from otx.algorithms.common.utils.ir import check_if_quantized @@ -76,7 +77,7 @@ IOptimizationTask, OptimizationType, ) -from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item, get_hierarchical_label_list +from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item logger = logging.getLogger(__name__) diff --git a/src/otx/algorithms/classification/task.py b/src/otx/algorithms/classification/task.py index 9faf4e931bf..bd0500a3ec1 100644 --- a/src/otx/algorithms/classification/task.py +++ b/src/otx/algorithms/classification/task.py @@ -28,6 +28,7 @@ get_cls_deploy_config, get_cls_inferencer_configuration, get_cls_model_api_configuration, + get_hierarchical_label_list, ) from otx.algorithms.classification.utils import ( get_multihead_class_info as get_hierarchical_info, @@ -75,7 +76,7 @@ from otx.api.serialization.label_mapper import label_schema_to_bytes from otx.api.usecases.evaluation.metrics_helper import MetricsHelper from otx.api.usecases.tasks.interfaces.export_interface import ExportType -from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item, get_hierarchical_label_list +from otx.api.utils.dataset_utils import add_saliency_maps_to_dataset_item from otx.api.utils.labels_utils import get_empty_label from otx.cli.utils.multi_gpu import is_multigpu_child_process diff --git a/src/otx/algorithms/classification/utils/__init__.py b/src/otx/algorithms/classification/utils/__init__.py index 533b871de17..536cd56bff7 100644 --- a/src/otx/algorithms/classification/utils/__init__.py +++ b/src/otx/algorithms/classification/utils/__init__.py @@ -8,10 +8,12 @@ get_cls_deploy_config, get_cls_inferencer_configuration, get_cls_model_api_configuration, + get_hierarchical_label_list, get_multihead_class_info, ) __all__ = [ + "get_hierarchical_label_list", "get_multihead_class_info", "get_cls_inferencer_configuration", "get_cls_deploy_config", diff --git a/src/otx/algorithms/classification/utils/cls_utils.py b/src/otx/algorithms/classification/utils/cls_utils.py index 23dc1ba1fa6..672381551e7 100644 --- a/src/otx/algorithms/classification/utils/cls_utils.py +++ b/src/otx/algorithms/classification/utils/cls_utils.py @@ -117,3 +117,24 @@ def get_cls_model_api_configuration(label_schema: LabelSchemaEntity, inference_c mapi_config[("model_info", "hierarchical_config")] = json.dumps(hierarchical_config) return mapi_config + + +def get_hierarchical_label_list(hierarchical_info, labels): + """Return hierarchical labels list which is adjusted to model outputs classes.""" + hierarchical_labels = [] + for head_idx in range(hierarchical_info["num_multiclass_heads"]): + logits_begin, logits_end = hierarchical_info["head_idx_to_logits_range"][str(head_idx)] + for logit in range(0, logits_end - logits_begin): + label_str = hierarchical_info["all_groups"][head_idx][logit] + label_idx = hierarchical_info["label_to_idx"][label_str] + hierarchical_labels.append(labels[label_idx]) + + if hierarchical_info["num_multilabel_classes"]: + logits_begin = hierarchical_info["num_single_label_classes"] + logits_end = len(labels) + for logit_idx, logit in enumerate(logits_end - logits_begin): + label_str_idx = hierarchical_info["num_multiclass_heads"] + logit_idx + label_str = hierarchical_info["all_groups"][label_str_idx][0] + label_idx = hierarchical_info["label_to_idx"][label_str] + hierarchical_labels.append(labels[label_idx]) + return hierarchical_labels diff --git a/src/otx/api/utils/dataset_utils.py b/src/otx/api/utils/dataset_utils.py index ad379a75b6a..7f0cddd6a74 100644 --- a/src/otx/api/utils/dataset_utils.py +++ b/src/otx/api/utils/dataset_utils.py @@ -272,15 +272,3 @@ def non_linear_normalization(saliency_map: np.ndarray) -> np.ndarray: saliency_map = 255.0 / (max_soft_score + 1e-12) * saliency_map return np.uint8(np.floor(saliency_map)) - - -def get_hierarchical_label_list(hierarchical_info, labels): - """Return hierarchical labels list which is adjusted to model outputs classes.""" - hierarchical_labels = [] - for head_idx in range(hierarchical_info["num_multiclass_heads"]): - logits_begin, logits_end = hierarchical_info["head_idx_to_logits_range"][str(head_idx)] - for logit in range(0, logits_end - logits_begin): - label_str = hierarchical_info["all_groups"][head_idx][logit] - otx_label = next(x for x in labels if x.name == label_str) - hierarchical_labels.append(otx_label) - return hierarchical_labels From 98e2027223502343032ef74cd10ab2d691e3c591 Mon Sep 17 00:00:00 2001 From: Galina Date: Mon, 21 Aug 2023 12:45:42 +0300 Subject: [PATCH 3/3] Fix unit tests --- src/otx/algorithms/classification/utils/cls_utils.py | 2 +- .../classification/tasks/test_classification_openvino_task.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/otx/algorithms/classification/utils/cls_utils.py b/src/otx/algorithms/classification/utils/cls_utils.py index 672381551e7..b1506ccc8e7 100644 --- a/src/otx/algorithms/classification/utils/cls_utils.py +++ b/src/otx/algorithms/classification/utils/cls_utils.py @@ -132,7 +132,7 @@ def get_hierarchical_label_list(hierarchical_info, labels): if hierarchical_info["num_multilabel_classes"]: logits_begin = hierarchical_info["num_single_label_classes"] logits_end = len(labels) - for logit_idx, logit in enumerate(logits_end - logits_begin): + for logit_idx, logit in enumerate(range(0, logits_end - logits_begin)): label_str_idx = hierarchical_info["num_multiclass_heads"] + logit_idx label_str = hierarchical_info["all_groups"][label_str_idx][0] label_idx = hierarchical_info["label_to_idx"][label_str] diff --git a/tests/unit/algorithms/classification/tasks/test_classification_openvino_task.py b/tests/unit/algorithms/classification/tasks/test_classification_openvino_task.py index c34c9c6ccbc..504265d4fd9 100644 --- a/tests/unit/algorithms/classification/tasks/test_classification_openvino_task.py +++ b/tests/unit/algorithms/classification/tasks/test_classification_openvino_task.py @@ -182,6 +182,7 @@ def test_explain(self, mocker): self.fake_input, ), ) + self.cls_ov_task.inferencer.model.hierarchical = False updpated_dataset = self.cls_ov_task.explain(self.dataset) assert updpated_dataset is not None