Fix conflict in changelog

CHANGELOG.md

@@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file.
 - Make semantic segmentation OpenVINO models compatible with ModelAPI (<https://github.com/openvinotoolkit/training_extensions/pull/2029>).
 - Support label hierarchy through LabelTree in LabelSchema for classification task (<https://github.com/openvinotoolkit/training_extensions/pull/2149>, <https://github.com/openvinotoolkit/training_extensions/pull/2152>).
 - Enhance exportable code file structure, video inference and default value for demo (<https://github.com/openvinotoolkit/training_extensions/pull/2051>).
+- Speedup OpenVINO inference in image classificaiton, semantic segmentation, object detection and instance segmentation tasks (<https://github.com/openvinotoolkit/training_extensions/pull/2105>).
 - Refactoring of ONNX export functionality (<https://github.com/openvinotoolkit/training_extensions/pull/2155>).
 
 ### Bug fixes

otx/algorithms/classification/adapters/openvino/task.py

@@ -19,8 +19,9 @@
 import logging
 import os
 import tempfile
+import time
 import warnings
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from zipfile import ZipFile
 
 import numpy as np
@@ -37,6 +38,7 @@
     get_cls_deploy_config,
     get_cls_inferencer_configuration,
 )
+from otx.algorithms.common.utils.utils import get_default_async_reqs_num
 from otx.api.entities.annotation import AnnotationSceneEntity
 from otx.api.entities.datasets import DatasetEntity
 from otx.api.entities.explain_parameters import ExplainParameters
@@ -108,18 +110,39 @@ def __init__(
 
         self.label_schema = label_schema
         model_adapter = OpenvinoAdapter(
-            create_core(), model_file, weight_file, device=device, max_num_requests=num_requests
+            create_core(),
+            model_file,
+            weight_file,
+            device=device,
+            max_num_requests=num_requests,
+            plugin_config={"PERFORMANCE_HINT": "THROUGHPUT"},
         )
         self.configuration = get_cls_inferencer_configuration(self.label_schema)
         self.model = Model.create_model("otx_classification", model_adapter, self.configuration, preload=True)
 
         self.converter = ClassificationToAnnotationConverter(self.label_schema)
+        self.callback_exceptions: List[Exception] = []
+        self.model.model_adapter.set_callback(self._async_callback)
 
     def pre_process(self, image: np.ndarray) -> Tuple[Dict[str, np.ndarray], Dict[str, Any]]:
         """Pre-process function of OpenVINO Classification Inferencer."""
 
         return self.model.preprocess(image)
 
+    def _async_callback(self, request: Any, callback_args: tuple) -> None:
+        """Fetches the results of async inference."""
+        try:
+            res_copy_func, args = callback_args
+            id, preprocessing_meta, result_handler = args
+            prediction = res_copy_func(request)
+
+            processed_prediciton = self.post_process(prediction, preprocessing_meta)
+            aux_data = self.model.postprocess_aux_outputs(prediction, preprocessing_meta)
+            result_handler(id, processed_prediciton, aux_data)
+
+        except Exception as e:
+            self.callback_exceptions.append(e)
+
     def post_process(
         self, prediction: Dict[str, np.ndarray], metadata: Dict[str, Any]
     ) -> Optional[AnnotationSceneEntity]:
@@ -138,6 +161,18 @@ def predict(self, image: np.ndarray) -> Tuple[AnnotationSceneEntity, np.ndarray,
 
         return predictions, probs, actmap, repr_vectors, act_score
 
+    def enqueue_prediction(self, image: np.ndarray, id: int, result_handler: Any) -> None:
+        """Runs async inference."""
+        if not self.model.is_ready():
+            self.model.await_any()
+        image, metadata = self.pre_process(image)
+        callback_data = id, metadata, result_handler
+        self.model.infer_async(image, callback_data)
+
+    def await_all(self) -> None:
+        """Await all running infer requests if any."""
+        self.model.await_all()
+
     def forward(self, image: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
         """Forward function of OpenVINO Classification Inferencer."""
 
@@ -187,6 +222,7 @@ def load_inferencer(self) -> ClassificationOpenVINOInferencer:
             self.task_environment.label_schema,
             self.model.get_data("openvino.xml"),
             self.model.get_data("openvino.bin"),
+            num_requests=get_default_async_reqs_num(),
         )
 
     # pylint: disable-msg=too-many-locals
@@ -199,15 +235,18 @@ def infer(
         dump_features = False
         process_saliency_maps = False
         explain_predicted_classes = True
+        enable_async_inference = True
+
         if inference_parameters is not None:
             update_progress_callback = inference_parameters.update_progress  # type: ignore
             dump_features = not inference_parameters.is_evaluation
             process_saliency_maps = inference_parameters.process_saliency_maps
             explain_predicted_classes = inference_parameters.explain_predicted_classes
+            enable_async_inference = inference_parameters.enable_async_inference
 
-        dataset_size = len(dataset)
-        for i, dataset_item in enumerate(dataset, 1):
-            predicted_scene, probs, saliency_map, repr_vector, act_score = self.inferencer.predict(dataset_item.numpy)
+        def add_prediction(id: int, predicted_scene: AnnotationSceneEntity, aux_data: tuple):
+            dataset_item = dataset[id]
+            probs, saliency_map, repr_vector, act_score = aux_data
             item_labels = predicted_scene.annotations[0].get_labels()
             dataset_item.append_labels(item_labels)
             active_score_media = FloatMetadata(name="active_score", value=act_score, float_type=FloatType.ACTIVE_SCORE)
@@ -235,7 +274,29 @@ def infer(
                         "Could not find Feature Vector and Saliency Map in OpenVINO output. "
                         "Please rerun OpenVINO export or retrain the model."
                     )
+
+        dataset_size = len(dataset)
+        total_time = 0.0
+        for i, dataset_item in enumerate(dataset, 1):
+            start_time = time.perf_counter()
+            if enable_async_inference:
+                self.inferencer.enqueue_prediction(dataset_item.numpy, i - 1, add_prediction)
+            else:
+                predicted_scene, probs, saliency_map, repr_vector, act_score = self.inferencer.predict(
+                    dataset_item.numpy
+                )
+                add_prediction(i - 1, predicted_scene, (probs, saliency_map, repr_vector, act_score))
+
+            end_time = time.perf_counter() - start_time
+            total_time += end_time
             update_progress_callback(int(i / dataset_size * 100))
+
+        self.inferencer.await_all()
+
+        logger.info(f"Avg time per image: {total_time/len(dataset)} secs")
+        logger.info(f"Total time: {total_time} secs")
+        logger.info("Classification OpenVINO inference completed")
+
         return dataset
 
     def explain(

otx/algorithms/common/utils/utils.py

@@ -116,3 +116,13 @@ def set_random_seed(seed, logger, deterministic=False):
     if deterministic:
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
+
+
+def get_default_async_reqs_num() -> int:
+    """Returns a default number of infer request for OV models."""
+    reqs_num = os.cpu_count()
+    if reqs_num is not None:
+        reqs_num = max(1, int(reqs_num / 2))
+        return reqs_num
+    else:
+        return 1

otx/algorithms/detection/adapters/mmdet/hooks/det_class_probability_map_hook.py

@@ -55,6 +55,7 @@ def func(
             cls_scores = feature_map
         else:
             cls_scores = self._get_cls_scores_from_feature_map(feature_map)
+        cls_scores = [torch.softmax(t, dim=1) for t in cls_scores]
 
         batch_size, _, height, width = cls_scores[-1].size()
         saliency_maps = torch.empty(batch_size, self._num_cls_out_channels, height, width)

otx/algorithms/detection/adapters/mmdet/models/detectors/custom_vfnet_detector.py

@@ -10,6 +10,8 @@
 
 from otx.algorithms.common.utils.logger import get_logger
 from otx.algorithms.common.utils.task_adapt import map_class_names
+from otx.algorithms.detection.adapters.mmdet.models.detectors.loss_dynamics_mixin import DetLossDynamicsTrackingMixin
+from otx.algorithms.detection.adapters.mmdet.models.loss_dyns import TrackingLossType
 
 from .l2sp_detector_mixin import L2SPDetectorMixin
 from .sam_detector_mixin import SAMDetectorMixin
@@ -21,9 +23,11 @@
 
 
 @DETECTORS.register_module()
-class CustomVFNet(SAMDetectorMixin, L2SPDetectorMixin, VFNet):
+class CustomVFNet(SAMDetectorMixin, DetLossDynamicsTrackingMixin, L2SPDetectorMixin, VFNet):
     """SAM optimizer & L2SP regularizer enabled custom VFNet."""
 
+    TRACKING_LOSS_TYPE = (TrackingLossType.cls, TrackingLossType.bbox, TrackingLossType.bbox_refine)
+
     def __init__(self, *args, task_adapt=None, **kwargs):
         super().__init__(*args, **kwargs)
 

otx/algorithms/detection/adapters/mmdet/models/detectors/custom_yolox_detector.py

@@ -18,6 +18,10 @@
 from otx.algorithms.detection.adapters.mmdet.hooks.det_class_probability_map_hook import (
     DetClassProbabilityMapHook,
 )
+from otx.algorithms.detection.adapters.mmdet.models.detectors.loss_dynamics_mixin import (
+    DetLossDynamicsTrackingMixin,
+)
+from otx.algorithms.detection.adapters.mmdet.models.loss_dyns import TrackingLossType
 
 from .l2sp_detector_mixin import L2SPDetectorMixin
 from .sam_detector_mixin import SAMDetectorMixin
@@ -29,9 +33,11 @@
 
 
 @DETECTORS.register_module()
-class CustomYOLOX(SAMDetectorMixin, L2SPDetectorMixin, YOLOX):
+class CustomYOLOX(SAMDetectorMixin, DetLossDynamicsTrackingMixin, L2SPDetectorMixin, YOLOX):
     """SAM optimizer & L2SP regularizer enabled custom YOLOX."""
 
+    TRACKING_LOSS_TYPE = (TrackingLossType.cls, TrackingLossType.bbox)
+
     def __init__(self, *args, task_adapt=None, **kwargs):
         super().__init__(*args, **kwargs)
 

otx/algorithms/detection/adapters/mmdet/models/heads/cross_dataset_detector_head.py

@@ -318,7 +318,7 @@ def wrapper(self, *args, **kwargs):
         return wrapper
 
     @staticmethod
-    def _wrap_get_targets(concatenate_last: bool = False):
+    def _wrap_get_targets(concatenate_last: bool = False, flatten: bool = False):
         def wrapper_with_option(func):
             signature = inspect.signature(func)
 
@@ -338,6 +338,8 @@ def wrapper(self, *args, **kwargs):
                 self.pos_assigned_gt_inds_list = []
                 targets = func(self, *args, **kwargs)
                 self.all_pos_assigned_gt_inds = images_to_levels(self.pos_assigned_gt_inds_list, num_level_anchors)
+                if flatten:
+                    self.all_pos_assigned_gt_inds = [gt_ind.reshape(-1) for gt_ind in self.all_pos_assigned_gt_inds]
                 if concatenate_last:
                     self.all_pos_assigned_gt_inds = torch.cat(self.all_pos_assigned_gt_inds, -1)
                 return targets