From 25c586d30b4c80569127bfa926a4d53d145a2efc Mon Sep 17 00:00:00 2001 From: Galina Date: Wed, 15 Feb 2023 21:46:03 +0200 Subject: [PATCH 01/10] Add a `dump_features` parameter for export --- .../model_wrappers/openvino_models.py | 9 ++++-- .../efficientnet_b0_cls_incr/deployment.py | 2 +- .../efficientnet_v2_s_cls_incr/deployment.py | 2 +- .../deployment.py | 2 +- .../classification/tasks/inference.py | 4 +-- .../classification/tasks/openvino.py | 31 ++++++++++++------- otx/algorithms/common/tasks/training_base.py | 6 ++++ .../detection/cspdarknet_yolox/deployment.py | 2 +- .../detection/mobilenetv2_atss/deployment.py | 2 +- .../detection/mobilenetv2_ssd/deployment.py | 2 +- otx/algorithms/detection/tasks/inference.py | 3 +- otx/cli/tools/export.py | 8 ++++- .../models/classifiers/sam_classifier.py | 10 ++++-- .../models/detectors/custom_atss_detector.py | 12 ++++--- .../detectors/custom_maskrcnn_detector.py | 10 ++++-- .../detectors/custom_single_stage_detector.py | 12 ++++--- .../models/detectors/custom_yolox_detector.py | 12 ++++--- 17 files changed, 87 insertions(+), 42 deletions(-) diff --git a/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py b/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py index 44dc287eaa2..dd38f3be477 100644 --- a/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py +++ b/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py @@ -103,8 +103,6 @@ def postprocess(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]): @check_input_parameters_type() def postprocess_aux_outputs(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]): """Post-process for auxiliary outputs.""" - saliency_map = outputs["saliency_map"][0] - repr_vector = outputs["feature_vector"].reshape(-1) logits = outputs[self.out_layer_name].squeeze() if self.multilabel: probs = sigmoid_numpy(logits) @@ -113,6 +111,13 @@ def postprocess_aux_outputs(self, outputs: Dict[str, np.ndarray], metadata: Dict else: probs = softmax_numpy(logits) act_score = float(np.max(probs) - np.min(probs)) + + if "saliency_map" in outputs: + saliency_map = outputs["saliency_map"][0] + repr_vector = outputs["feature_vector"].reshape(-1) + else: + saliency_map, repr_vector = None, None + return probs, saliency_map, repr_vector, act_score diff --git a/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/deployment.py b/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/deployment.py index 421059d07f0..27ac2597fdd 100644 --- a/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/deployment.py +++ b/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../base/deployments/base_classification_dynamic.py"] ir_config = dict( - output_names=["logits", "feature_vector", "saliency_map"], + output_names=["logits"], ) backend_config = dict( diff --git a/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/deployment.py b/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/deployment.py index 6ab4d2e2d5e..f4b171c8bd4 100644 --- a/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/deployment.py +++ b/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../base/deployments/base_classification_dynamic.py"] ir_config = dict( - output_names=["logits", "feature_vector", "saliency_map"], + output_names=["logits"], ) backend_config = dict( diff --git a/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/deployment.py b/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/deployment.py index b57137aed3c..4af7a810348 100644 --- a/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/deployment.py +++ b/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../base/deployments/base_classification_dynamic.py"] ir_config = dict( - output_names=["logits", "feature_vector", "saliency_map"], + output_names=["logits"], ) backend_config = dict( diff --git a/otx/algorithms/classification/tasks/inference.py b/otx/algorithms/classification/tasks/inference.py index 3a655093ed5..3580556dc04 100644 --- a/otx/algorithms/classification/tasks/inference.py +++ b/otx/algorithms/classification/tasks/inference.py @@ -195,7 +195,7 @@ def unload(self): self.cleanup() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity): + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = False): """Export function of OTX Classification Task.""" logger.info("Exporting the model") @@ -205,7 +205,7 @@ def export(self, export_type: ExportType, output_model: ModelEntity): output_model.optimization_type = ModelOptimizationType.MO stage_module = "ClsExporter" - results = self._run_task(stage_module, mode="train", export=True) + results = self._run_task(stage_module, mode="train", export=True, dump_features=dump_features) outputs = results.get("outputs") logger.debug(f"results of run_task = {outputs}") if outputs is None: diff --git a/otx/algorithms/classification/tasks/openvino.py b/otx/algorithms/classification/tasks/openvino.py index cb9bea3dac5..6cf60501afa 100644 --- a/otx/algorithms/classification/tasks/openvino.py +++ b/otx/algorithms/classification/tasks/openvino.py @@ -19,6 +19,7 @@ import logging import os import tempfile +import warnings from typing import Any, Dict, Optional, Tuple, Union from zipfile import ZipFile @@ -78,8 +79,6 @@ from openvino.model_zoo.model_api.adapters import OpenvinoAdapter, create_core from openvino.model_zoo.model_api.models import Model except ImportError: - import warnings - warnings.warn("ModelAPI was not found.") logger = logging.getLogger(__name__) @@ -234,17 +233,25 @@ def infer( probs_meta = TensorEntity(name="probabilities", numpy=probs.reshape(-1)) dataset_item.append_metadata_item(probs_meta, model=self.model) - feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1)) - dataset_item.append_metadata_item(feature_vec_media, model=self.model) if dump_features: - add_saliency_maps_to_dataset_item( - dataset_item=dataset_item, - saliency_map=saliency_map, - model=self.model, - labels=self.task_environment.get_labels(), - task="cls", - predicted_scene=predicted_scene, - ) + if saliency_map is not None and repr_vector is not None: + feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1)) + dataset_item.append_metadata_item(feature_vec_media, model=self.model) + + add_saliency_maps_to_dataset_item( + dataset_item=dataset_item, + saliency_map=saliency_map, + model=self.model, + labels=self.task_environment.get_labels(), + task="cls", + predicted_scene=predicted_scene, + ) + else: + warnings.warn( + "Could not find Feature Vector and Saliency Map in OpenVINO output. " + "Please rerun OpenVINO export or retrain the model." + ) + update_progress_callback(int(i / dataset_size * 100)) return dataset diff --git a/otx/algorithms/common/tasks/training_base.py b/otx/algorithms/common/tasks/training_base.py index 5ab1f6289da..405a84dd53b 100644 --- a/otx/algorithms/common/tasks/training_base.py +++ b/otx/algorithms/common/tasks/training_base.py @@ -332,6 +332,12 @@ def _initialize(self, options=None): # noqa: C901 assert len(self._precision) == 1 options["precision"] = str(self._precision[0]) + options["deploy_cfg"]["dump_features"] = options["dump_features"] + if options["dump_features"]: + output_names = options["deploy_cfg"]["ir_config"]["output_names"] + if "feature_vector" not in output_names and "saliency_map" not in output_names: + options["deploy_cfg"]["ir_config"]["output_names"] += ["feature_vector", "saliency_map"] + self._initialize_post_hook(options) logger.info("initialized.") diff --git a/otx/algorithms/detection/configs/detection/cspdarknet_yolox/deployment.py b/otx/algorithms/detection/configs/detection/cspdarknet_yolox/deployment.py index b7f3953101d..fe3d750dd7e 100644 --- a/otx/algorithms/detection/configs/detection/cspdarknet_yolox/deployment.py +++ b/otx/algorithms/detection/configs/detection/cspdarknet_yolox/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../../base/deployments/base_detection_dynamic.py"] ir_config = dict( - output_names=["boxes", "labels", "feature_vector", "saliency_map"], + output_names=["boxes", "labels"], ) backend_config = dict( diff --git a/otx/algorithms/detection/configs/detection/mobilenetv2_atss/deployment.py b/otx/algorithms/detection/configs/detection/mobilenetv2_atss/deployment.py index ce46c74465f..2004bde9ef5 100644 --- a/otx/algorithms/detection/configs/detection/mobilenetv2_atss/deployment.py +++ b/otx/algorithms/detection/configs/detection/mobilenetv2_atss/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../../base/deployments/base_detection_dynamic.py"] ir_config = dict( - output_names=["boxes", "labels", "feature_vector", "saliency_map"], + output_names=["boxes", "labels"], ) backend_config = dict( diff --git a/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/deployment.py b/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/deployment.py index c11deeb7db3..68378ad83d4 100644 --- a/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/deployment.py +++ b/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/deployment.py @@ -3,7 +3,7 @@ _base_ = ["../../base/deployments/base_detection_dynamic.py"] ir_config = dict( - output_names=["boxes", "labels", "feature_vector", "saliency_map"], + output_names=["boxes", "labels"], ) backend_config = dict( diff --git a/otx/algorithms/detection/tasks/inference.py b/otx/algorithms/detection/tasks/inference.py index e5c9ff7d2bc..3d630b8d0b6 100644 --- a/otx/algorithms/detection/tasks/inference.py +++ b/otx/algorithms/detection/tasks/inference.py @@ -227,7 +227,7 @@ def unload(self): self.cleanup() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity): + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = False): """Export function of OTX Detection Task.""" # copied from OTX inference_task.py logger.info("Exporting the model") @@ -241,6 +241,7 @@ def export(self, export_type: ExportType, output_model: ModelEntity): stage_module, mode="train", export=True, + dump_features=dump_features, ) outputs = results.get("outputs") logger.debug(f"results of run_task = {outputs}") diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index 0a4cf97939b..4e0a5ba24c8 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -40,6 +40,11 @@ def get_args(): "--save-model-to", help="Location where exported model will be stored.", ) + parser.add_argument( + "--dump_features", + action="store_true", + help="Whether to return feature vector and saliency map for explanation purposes.", + ) return parser.parse_args() @@ -83,8 +88,9 @@ def main(): task = task_class(task_environment=environment) exported_model = ModelEntity(None, environment.get_model_configuration()) + # args.dump_features = True - task.export(ExportType.OPENVINO, exported_model) + task.export(ExportType.OPENVINO, exported_model, args.dump_features) if "save_model_to" not in args or not args.save_model_to: args.save_model_to = str(config_manager.workspace_root / "model-exported") diff --git a/otx/mpa/modules/models/classifiers/sam_classifier.py b/otx/mpa/modules/models/classifiers/sam_classifier.py index 27d1d2cead0..36c98ba6e98 100644 --- a/otx/mpa/modules/models/classifiers/sam_classifier.py +++ b/otx/mpa/modules/models/classifiers/sam_classifier.py @@ -295,6 +295,10 @@ def sam_image_classifier__extract_feat(ctx, self, img): def sam_image_classifier__simple_test(ctx, self, img, img_metas): feat, backbone_feat = self.extract_feat(img) logit = self.head.simple_test(feat) - saliency_map = ReciproCAMHook(self).func(backbone_feat) - feature_vector = FeatureVectorHook.func(backbone_feat) - return logit, feature_vector, saliency_map + + if ctx.cfg["dump_features"]: + saliency_map = ReciproCAMHook(self).func(backbone_feat) + feature_vector = FeatureVectorHook.func(backbone_feat) + return logit, feature_vector, saliency_map + + return logit diff --git a/otx/mpa/modules/models/detectors/custom_atss_detector.py b/otx/mpa/modules/models/detectors/custom_atss_detector.py index 5b175c6f421..2dcc297419f 100644 --- a/otx/mpa/modules/models/detectors/custom_atss_detector.py +++ b/otx/mpa/modules/models/detectors/custom_atss_detector.py @@ -86,10 +86,14 @@ def custom_atss__simple_test(ctx, self, img, img_metas, **kwargs): feat = self.extract_feat(img) outs = self.bbox_head(feat) bbox_results = self.bbox_head.get_bboxes(*outs, img_metas=img_metas, cfg=self.test_cfg, **kwargs) - feature_vector = FeatureVectorHook.func(feat) - cls_scores = outs[0] - saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) - return (*bbox_results, feature_vector, saliency_map) + + if ctx.cfg["dump_features"]: + feature_vector = FeatureVectorHook.func(feat) + cls_scores = outs[0] + saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) + return (*bbox_results, feature_vector, saliency_map) + + return bbox_results @mark("custom_atss_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) def __forward_impl(ctx, self, img, img_metas, **kwargs): diff --git a/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py b/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py index 0bb38a5bac6..fe3053da967 100644 --- a/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py +++ b/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py @@ -91,12 +91,16 @@ def load_state_dict_pre_hook(model, model_classes, chkpt_classes, chkpt_dict, pr def custom_mask_rcnn__simple_test(ctx, self, img, img_metas, proposals=None, **kwargs): assert self.with_bbox, "Bbox head must be implemented." x = self.extract_feat(img) - feature_vector = FeatureVectorHook.func(x) - saliency_map = ActivationMapHook.func(x[-1]) if proposals is None: proposals, _ = self.rpn_head.simple_test_rpn(x, img_metas) out = self.roi_head.simple_test(x, proposals, img_metas, rescale=False) - return (*out, feature_vector, saliency_map) + + if ctx.cfg["dump_features"]: + feature_vector = FeatureVectorHook.func(x) + saliency_map = ActivationMapHook.func(x[-1]) + return (*out, feature_vector, saliency_map) + + return out @mark("custom_maskrcnn_forward", inputs=["input"], outputs=["dets", "labels", "masks", "feats", "saliencies"]) def __forward_impl(ctx, self, img, img_metas, **kwargs): diff --git a/otx/mpa/modules/models/detectors/custom_single_stage_detector.py b/otx/mpa/modules/models/detectors/custom_single_stage_detector.py index bff6d8b446d..609646bcaeb 100644 --- a/otx/mpa/modules/models/detectors/custom_single_stage_detector.py +++ b/otx/mpa/modules/models/detectors/custom_single_stage_detector.py @@ -142,10 +142,14 @@ def custom_single_stage_detector__simple_test(ctx, self, img, img_metas, **kwarg feat = self.extract_feat(img) outs = self.bbox_head(feat) bbox_results = self.bbox_head.get_bboxes(*outs, img_metas=img_metas, cfg=self.test_cfg, **kwargs) - feature_vector = FeatureVectorHook.func(feat) - cls_scores = outs[0] - saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) - return (*bbox_results, feature_vector, saliency_map) + + if ctx.cfg["dump_features"]: + feature_vector = FeatureVectorHook.func(feat) + cls_scores = outs[0] + saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) + return (*bbox_results, feature_vector, saliency_map) + + return bbox_results @mark("custom_ssd_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) def __forward_impl(ctx, self, img, img_metas, **kwargs): diff --git a/otx/mpa/modules/models/detectors/custom_yolox_detector.py b/otx/mpa/modules/models/detectors/custom_yolox_detector.py index 665f1fbbfe4..e48973e72e4 100644 --- a/otx/mpa/modules/models/detectors/custom_yolox_detector.py +++ b/otx/mpa/modules/models/detectors/custom_yolox_detector.py @@ -126,10 +126,14 @@ def custom_yolox__simple_test(ctx, self, img, img_metas, **kwargs): feat = self.extract_feat(img) outs = self.bbox_head(feat) bbox_results = self.bbox_head.get_bboxes(*outs, img_metas=img_metas, cfg=self.test_cfg, **kwargs) - feature_vector = FeatureVectorHook.func(feat) - cls_scores = outs[0] - saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) - return (*bbox_results, feature_vector, saliency_map) + + if ctx.cfg["dump_features"]: + feature_vector = FeatureVectorHook.func(feat) + cls_scores = outs[0] + saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) + return (*bbox_results, feature_vector, saliency_map) + + return bbox_results @mark("custom_yolox_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) def __forward_impl(ctx, self, img, img_metas, **kwargs): From bbcf31ddb8bedde89dbffdc001b17823485c03b7 Mon Sep 17 00:00:00 2001 From: Galina Date: Sun, 19 Feb 2023 22:53:04 +0200 Subject: [PATCH 02/10] Fix pre-commit & add parameter to IExport --- .../adapters/openvino/model_wrappers/openvino_models.py | 2 +- otx/api/usecases/tasks/interfaces/export_interface.py | 3 ++- otx/cli/tools/export.py | 1 - otx/mpa/modules/models/classifiers/sam_classifier.py | 2 +- otx/mpa/modules/models/detectors/custom_atss_detector.py | 2 +- otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py | 2 +- .../modules/models/detectors/custom_single_stage_detector.py | 2 +- otx/mpa/modules/models/detectors/custom_yolox_detector.py | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py b/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py index eb11279686d..5ee6942d65d 100644 --- a/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py +++ b/otx/algorithms/classification/adapters/openvino/model_wrappers/openvino_models.py @@ -117,7 +117,7 @@ def postprocess_aux_outputs(self, outputs: Dict[str, np.ndarray], metadata: Dict repr_vector = outputs["feature_vector"].reshape(-1) else: saliency_map, repr_vector = None, None - + return probs, saliency_map, repr_vector, act_score diff --git a/otx/api/usecases/tasks/interfaces/export_interface.py b/otx/api/usecases/tasks/interfaces/export_interface.py index e206a6a6f20..82a2d2db7d0 100644 --- a/otx/api/usecases/tasks/interfaces/export_interface.py +++ b/otx/api/usecases/tasks/interfaces/export_interface.py @@ -20,11 +20,12 @@ class IExportTask(metaclass=abc.ABCMeta): """A base interface class for tasks which can export their models.""" @abc.abstractmethod - def export(self, export_type: ExportType, output_model: ModelEntity): + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = False): """This method defines the interface for export. Args: export_type (ExportType): The type of optimization. output_model (ModelEntity): The output model entity. + dump_features (bool): Flag to return "feature_vector" and "saliency_map". """ raise NotImplementedError diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index 4e0a5ba24c8..abd448eab97 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -88,7 +88,6 @@ def main(): task = task_class(task_environment=environment) exported_model = ModelEntity(None, environment.get_model_configuration()) - # args.dump_features = True task.export(ExportType.OPENVINO, exported_model, args.dump_features) diff --git a/otx/mpa/modules/models/classifiers/sam_classifier.py b/otx/mpa/modules/models/classifiers/sam_classifier.py index 36c98ba6e98..8edbffda75e 100644 --- a/otx/mpa/modules/models/classifiers/sam_classifier.py +++ b/otx/mpa/modules/models/classifiers/sam_classifier.py @@ -300,5 +300,5 @@ def sam_image_classifier__simple_test(ctx, self, img, img_metas): saliency_map = ReciproCAMHook(self).func(backbone_feat) feature_vector = FeatureVectorHook.func(backbone_feat) return logit, feature_vector, saliency_map - + return logit diff --git a/otx/mpa/modules/models/detectors/custom_atss_detector.py b/otx/mpa/modules/models/detectors/custom_atss_detector.py index 2dcc297419f..dbf416a48d2 100644 --- a/otx/mpa/modules/models/detectors/custom_atss_detector.py +++ b/otx/mpa/modules/models/detectors/custom_atss_detector.py @@ -92,7 +92,7 @@ def custom_atss__simple_test(ctx, self, img, img_metas, **kwargs): cls_scores = outs[0] saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) return (*bbox_results, feature_vector, saliency_map) - + return bbox_results @mark("custom_atss_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) diff --git a/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py b/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py index fe3053da967..2a840cd2880 100644 --- a/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py +++ b/otx/mpa/modules/models/detectors/custom_maskrcnn_detector.py @@ -99,7 +99,7 @@ def custom_mask_rcnn__simple_test(ctx, self, img, img_metas, proposals=None, **k feature_vector = FeatureVectorHook.func(x) saliency_map = ActivationMapHook.func(x[-1]) return (*out, feature_vector, saliency_map) - + return out @mark("custom_maskrcnn_forward", inputs=["input"], outputs=["dets", "labels", "masks", "feats", "saliencies"]) diff --git a/otx/mpa/modules/models/detectors/custom_single_stage_detector.py b/otx/mpa/modules/models/detectors/custom_single_stage_detector.py index 609646bcaeb..15d02b6e226 100644 --- a/otx/mpa/modules/models/detectors/custom_single_stage_detector.py +++ b/otx/mpa/modules/models/detectors/custom_single_stage_detector.py @@ -148,7 +148,7 @@ def custom_single_stage_detector__simple_test(ctx, self, img, img_metas, **kwarg cls_scores = outs[0] saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) return (*bbox_results, feature_vector, saliency_map) - + return bbox_results @mark("custom_ssd_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) diff --git a/otx/mpa/modules/models/detectors/custom_yolox_detector.py b/otx/mpa/modules/models/detectors/custom_yolox_detector.py index e48973e72e4..e7b0f2774f3 100644 --- a/otx/mpa/modules/models/detectors/custom_yolox_detector.py +++ b/otx/mpa/modules/models/detectors/custom_yolox_detector.py @@ -126,13 +126,13 @@ def custom_yolox__simple_test(ctx, self, img, img_metas, **kwargs): feat = self.extract_feat(img) outs = self.bbox_head(feat) bbox_results = self.bbox_head.get_bboxes(*outs, img_metas=img_metas, cfg=self.test_cfg, **kwargs) - + if ctx.cfg["dump_features"]: feature_vector = FeatureVectorHook.func(feat) cls_scores = outs[0] saliency_map = DetSaliencyMapHook(self).func(cls_scores, cls_scores_provided=True) return (*bbox_results, feature_vector, saliency_map) - + return bbox_results @mark("custom_yolox_forward", inputs=["input"], outputs=["dets", "labels", "feats", "saliencies"]) From aeb63386d73e1e984ad00ad66e5b11e9e9041cf3 Mon Sep 17 00:00:00 2001 From: Galina Date: Mon, 20 Feb 2023 10:42:56 +0200 Subject: [PATCH 03/10] Add dump_features=True to other tasks' export --- otx/algorithms/action/tasks/inference.py | 2 +- otx/algorithms/anomaly/tasks/inference.py | 2 +- otx/algorithms/segmentation/tasks/inference.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/otx/algorithms/action/tasks/inference.py b/otx/algorithms/action/tasks/inference.py index 10cd3beae02..ccdccc61539 100644 --- a/otx/algorithms/action/tasks/inference.py +++ b/otx/algorithms/action/tasks/inference.py @@ -307,7 +307,7 @@ def unload(self): self._delete_scratch_space() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity): + def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs): """Export function of OTX Action Task.""" # copied from OTX inference_task.py logger.info("Exporting the model") diff --git a/otx/algorithms/anomaly/tasks/inference.py b/otx/algorithms/anomaly/tasks/inference.py index c879ee3380c..4bd4a66590c 100644 --- a/otx/algorithms/anomaly/tasks/inference.py +++ b/otx/algorithms/anomaly/tasks/inference.py @@ -247,7 +247,7 @@ def _export_to_onnx(self, onnx_path: str): opset_version=11, ) - def export(self, export_type: ExportType, output_model: ModelEntity) -> None: + def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs) -> None: """Export model to OpenVINO IR. Args: diff --git a/otx/algorithms/segmentation/tasks/inference.py b/otx/algorithms/segmentation/tasks/inference.py index 2066286c372..36ca2692915 100644 --- a/otx/algorithms/segmentation/tasks/inference.py +++ b/otx/algorithms/segmentation/tasks/inference.py @@ -147,7 +147,7 @@ def unload(self): self.cleanup() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity): + def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs): """Export function of OTX Segmentation Task.""" logger.info("Exporting the model") if export_type != ExportType.OPENVINO: From 5de193dd4080117a2e850e56345de3b1cd95a242 Mon Sep 17 00:00:00 2001 From: Galina Date: Mon, 20 Feb 2023 15:21:32 +0200 Subject: [PATCH 04/10] Add comments --- otx/algorithms/action/tasks/inference.py | 3 ++- otx/algorithms/anomaly/tasks/inference.py | 4 +++- otx/algorithms/segmentation/tasks/inference.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/otx/algorithms/action/tasks/inference.py b/otx/algorithms/action/tasks/inference.py index ccdccc61539..aa1ca936def 100644 --- a/otx/algorithms/action/tasks/inference.py +++ b/otx/algorithms/action/tasks/inference.py @@ -307,8 +307,9 @@ def unload(self): self._delete_scratch_space() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs): + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = True): """Export function of OTX Action Task.""" + # TODO: add dumping saliency maps and representation vectors according to dump_features flag # copied from OTX inference_task.py logger.info("Exporting the model") if export_type != ExportType.OPENVINO: diff --git a/otx/algorithms/anomaly/tasks/inference.py b/otx/algorithms/anomaly/tasks/inference.py index 4bd4a66590c..7c1cfcb4bfa 100644 --- a/otx/algorithms/anomaly/tasks/inference.py +++ b/otx/algorithms/anomaly/tasks/inference.py @@ -247,16 +247,18 @@ def _export_to_onnx(self, onnx_path: str): opset_version=11, ) - def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs) -> None: + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = True) -> None: """Export model to OpenVINO IR. Args: export_type (ExportType): Export type should be ExportType.OPENVINO output_model (ModelEntity): The model entity in which to write the OpenVINO IR data + dump_features (bool): Flag to return "feature_vector" and "saliency_map". Raises: Exception: If export_type is not ExportType.OPENVINO """ + # TODO: add dumping saliency maps and representation vectors according to dump_features flag assert export_type == ExportType.OPENVINO, f"Incorrect export_type={export_type}" output_model.model_format = ModelFormat.OPENVINO output_model.optimization_type = ModelOptimizationType.MO diff --git a/otx/algorithms/segmentation/tasks/inference.py b/otx/algorithms/segmentation/tasks/inference.py index 36ca2692915..7051760b97a 100644 --- a/otx/algorithms/segmentation/tasks/inference.py +++ b/otx/algorithms/segmentation/tasks/inference.py @@ -147,7 +147,7 @@ def unload(self): self.cleanup() @check_input_parameters_type() - def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs): + def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = True): """Export function of OTX Segmentation Task.""" logger.info("Exporting the model") if export_type != ExportType.OPENVINO: @@ -160,6 +160,7 @@ def export(self, export_type: ExportType, output_model: ModelEntity, **kwargs): stage_module, mode="train", export=True, + dump_features=dump_features, ) outputs = results.get("outputs") logger.debug(f"results of run_task = {outputs}") From ff052b18f127b8bd84b8eae7cc201b9c0cf856fb Mon Sep 17 00:00:00 2001 From: Galina Date: Mon, 20 Feb 2023 15:57:38 +0200 Subject: [PATCH 05/10] Fix pre-commit --- tests/unit/cli/tools/test_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cli/tools/test_export.py b/tests/unit/cli/tools/test_export.py index f9e2e53aff2..449f0ef2a9b 100644 --- a/tests/unit/cli/tools/test_export.py +++ b/tests/unit/cli/tools/test_export.py @@ -65,7 +65,7 @@ def test_main(mocker, mock_args, mock_task, mock_config_manager, tmp_dir): mocker.patch.object(target_package, "read_label_schema") mocker.patch.object(target_package, "read_binary") - def mock_export_side_effect(export_type, output_model): + def mock_export_side_effect(export_type, output_model, dump_features): output_model.set_data("fake.xml", b"fake") mock_task.export.side_effect = mock_export_side_effect From 2af2599d97b2c48059fb9c43841c560c7988e672 Mon Sep 17 00:00:00 2001 From: Galina Date: Mon, 20 Feb 2023 17:31:01 +0200 Subject: [PATCH 06/10] Add NotImplementedError --- otx/algorithms/action/tasks/inference.py | 3 +++ otx/algorithms/anomaly/tasks/inference.py | 3 +++ otx/algorithms/segmentation/tasks/inference.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/otx/algorithms/action/tasks/inference.py b/otx/algorithms/action/tasks/inference.py index aa1ca936def..e43396f42d7 100644 --- a/otx/algorithms/action/tasks/inference.py +++ b/otx/algorithms/action/tasks/inference.py @@ -310,6 +310,9 @@ def unload(self): def export(self, export_type: ExportType, output_model: ModelEntity, dump_features: bool = True): """Export function of OTX Action Task.""" # TODO: add dumping saliency maps and representation vectors according to dump_features flag + if not dump_features: + raise NotImplementedError("Ommitting feature dumping is not implemented.") + # copied from OTX inference_task.py logger.info("Exporting the model") if export_type != ExportType.OPENVINO: diff --git a/otx/algorithms/anomaly/tasks/inference.py b/otx/algorithms/anomaly/tasks/inference.py index 7c1cfcb4bfa..08f553d4a3a 100644 --- a/otx/algorithms/anomaly/tasks/inference.py +++ b/otx/algorithms/anomaly/tasks/inference.py @@ -259,6 +259,9 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur Exception: If export_type is not ExportType.OPENVINO """ # TODO: add dumping saliency maps and representation vectors according to dump_features flag + if not dump_features: + raise NotImplementedError("Ommitting feature dumping is not implemented.") + assert export_type == ExportType.OPENVINO, f"Incorrect export_type={export_type}" output_model.model_format = ModelFormat.OPENVINO output_model.optimization_type = ModelOptimizationType.MO diff --git a/otx/algorithms/segmentation/tasks/inference.py b/otx/algorithms/segmentation/tasks/inference.py index 7051760b97a..b9802751ea2 100644 --- a/otx/algorithms/segmentation/tasks/inference.py +++ b/otx/algorithms/segmentation/tasks/inference.py @@ -154,6 +154,9 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur raise RuntimeError(f"not supported export type {export_type}") output_model.model_format = ModelFormat.OPENVINO output_model.optimization_type = ModelOptimizationType.MO + # TODO: add dumping saliency maps and representation vectors according to dump_features flag + if not dump_features: + raise NotImplementedError("Ommitting feature dumping is not implemented.") stage_module = "SegExporter" results = self._run_task( From 587d26601bf2f4e51f36a4852310f4e39867a32c Mon Sep 17 00:00:00 2001 From: Galina Date: Tue, 21 Feb 2023 11:50:09 +0200 Subject: [PATCH 07/10] Fix CLI tests --- otx/algorithms/action/tasks/inference.py | 5 ++++- otx/algorithms/anomaly/tasks/inference.py | 5 ++++- otx/algorithms/segmentation/tasks/inference.py | 4 +++- tests/test_suite/run_test_command.py | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/otx/algorithms/action/tasks/inference.py b/otx/algorithms/action/tasks/inference.py index e43396f42d7..a534d1d1331 100644 --- a/otx/algorithms/action/tasks/inference.py +++ b/otx/algorithms/action/tasks/inference.py @@ -311,7 +311,10 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur """Export function of OTX Action Task.""" # TODO: add dumping saliency maps and representation vectors according to dump_features flag if not dump_features: - raise NotImplementedError("Ommitting feature dumping is not implemented.") + logger.warning( + "Ommitting feature dumping is not implemented." + "The saliency maps and representation vector outputs will be dumped in the exported model." + ) # copied from OTX inference_task.py logger.info("Exporting the model") diff --git a/otx/algorithms/anomaly/tasks/inference.py b/otx/algorithms/anomaly/tasks/inference.py index 08f553d4a3a..c12914bb34a 100644 --- a/otx/algorithms/anomaly/tasks/inference.py +++ b/otx/algorithms/anomaly/tasks/inference.py @@ -260,7 +260,10 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur """ # TODO: add dumping saliency maps and representation vectors according to dump_features flag if not dump_features: - raise NotImplementedError("Ommitting feature dumping is not implemented.") + logger.warning( + "Ommitting feature dumping is not implemented." + "The saliency maps and representation vector outputs will be dumped in the exported model." + ) assert export_type == ExportType.OPENVINO, f"Incorrect export_type={export_type}" output_model.model_format = ModelFormat.OPENVINO diff --git a/otx/algorithms/segmentation/tasks/inference.py b/otx/algorithms/segmentation/tasks/inference.py index b9802751ea2..1c820341ec6 100644 --- a/otx/algorithms/segmentation/tasks/inference.py +++ b/otx/algorithms/segmentation/tasks/inference.py @@ -156,7 +156,9 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur output_model.optimization_type = ModelOptimizationType.MO # TODO: add dumping saliency maps and representation vectors according to dump_features flag if not dump_features: - raise NotImplementedError("Ommitting feature dumping is not implemented.") + logger.warning( + "Ommitting feature dumping is not implemented. The saliency maps and representation vector outputs will be dumped in exported model." + ) stage_module = "SegExporter" results = self._run_task( diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index 02d2b11e2cd..b1b69bb2616 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -209,6 +209,7 @@ def otx_export_testing(template, root): f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", "--save-model-to", f"{template_work_dir}/exported_{template.model_template_id}", + f"--dump_features" ] check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml") From 23b145fad61cc35a48816576556f9934d5a4cf0e Mon Sep 17 00:00:00 2001 From: Galina Date: Tue, 21 Feb 2023 12:17:09 +0200 Subject: [PATCH 08/10] Solve merge conflict --- .../classification/tasks/openvino.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/otx/algorithms/classification/tasks/openvino.py b/otx/algorithms/classification/tasks/openvino.py index a03604f4859..0a55c31a556 100644 --- a/otx/algorithms/classification/tasks/openvino.py +++ b/otx/algorithms/classification/tasks/openvino.py @@ -239,15 +239,24 @@ def infer( dataset_item.append_metadata_item(probs_meta, model=self.model) if dump_features: - add_saliency_maps_to_dataset_item( - dataset_item=dataset_item, - saliency_map=saliency_map, - model=self.model, - labels=self.task_environment.get_labels(), - predicted_scored_labels=item_labels, - explain_predicted_classes=explain_predicted_classes, - process_saliency_maps=process_saliency_maps, - ) + if saliency_map is not None and repr_vector is not None: + feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1)) + dataset_item.append_metadata_item(feature_vec_media, model=self.model) + + add_saliency_maps_to_dataset_item( + dataset_item=dataset_item, + saliency_map=saliency_map, + model=self.model, + labels=self.task_environment.get_labels(), + predicted_scored_labels=item_labels, + explain_predicted_classes=explain_predicted_classes, + process_saliency_maps=process_saliency_maps, + ) + else: + warnings.warn( + "Could not find Feature Vector and Saliency Map in OpenVINO output. " + "Please rerun OpenVINO export or retrain the model." + ) update_progress_callback(int(i / dataset_size * 100)) return dataset From cb3fc2d354db6213f8d7bc3611a7508f6350d5f2 Mon Sep 17 00:00:00 2001 From: Galina Date: Tue, 21 Feb 2023 12:26:07 +0200 Subject: [PATCH 09/10] Fix pre-commit --- tests/test_suite/run_test_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index b1b69bb2616..52b1b092064 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -209,7 +209,7 @@ def otx_export_testing(template, root): f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", "--save-model-to", f"{template_work_dir}/exported_{template.model_template_id}", - f"--dump_features" + f"--dump_features", ] check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml") From ef21b8c6a413f44cae27243b459e34d8b58d23d4 Mon Sep 17 00:00:00 2001 From: Galina Date: Tue, 21 Feb 2023 12:52:34 +0200 Subject: [PATCH 10/10] Fix pre-commit --- otx/algorithms/segmentation/tasks/inference.py | 7 ++++++- tests/test_suite/run_test_command.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/otx/algorithms/segmentation/tasks/inference.py b/otx/algorithms/segmentation/tasks/inference.py index 759341894b5..089ea9e611c 100644 --- a/otx/algorithms/segmentation/tasks/inference.py +++ b/otx/algorithms/segmentation/tasks/inference.py @@ -24,6 +24,7 @@ patch_data_pipeline, patch_default_config, patch_runner, + remove_from_configs_by_type, ) from otx.algorithms.common.configs import TrainType from otx.algorithms.common.tasks import BaseTask @@ -87,6 +88,7 @@ class SegmentationInferenceTask(BaseTask, IInferenceTask, IExportTask, IEvaluati @check_input_parameters_type() def __init__(self, task_environment: TaskEnvironment, **kwargs): # self._should_stop = False + self.freeze = True self.metric = "mDice" self._label_dictionary = {} # type: Dict @@ -155,7 +157,8 @@ def export(self, export_type: ExportType, output_model: ModelEntity, dump_featur # TODO: add dumping saliency maps and representation vectors according to dump_features flag if not dump_features: logger.warning( - "Ommitting feature dumping is not implemented. The saliency maps and representation vector outputs will be dumped in exported model." + "Ommitting feature dumping is not implemented." + "The saliency maps and representation vector outputs will be dumped in the exported model." ) stage_module = "SegExporter" @@ -218,6 +221,8 @@ def _init_recipe(self): if self._recipe_cfg.get("override_configs", None): self.override_configs.update(self._recipe_cfg.override_configs) + if not self.freeze: + remove_from_configs_by_type(self._recipe_cfg.custom_hooks, "FreezeLayers") logger.info(f"initialized recipe = {recipe}") def _update_stage_module(self, stage_module: str): diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index 52b1b092064..3b9110c09f8 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -209,7 +209,7 @@ def otx_export_testing(template, root): f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", "--save-model-to", f"{template_work_dir}/exported_{template.model_template_id}", - f"--dump_features", + "--dump_features", ] check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml")