From 08a9881b2745cf1dc8a91c5bdc4713c2172090eb Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 20 Apr 2023 11:20:05 +0100 Subject: [PATCH 1/8] draft --- .../detection/adapters/openvino/task.py | 2 ++ .../instance_segmentation/configuration.yaml | 19 +++++++++++++++++++ .../efficientnetb2b_maskrcnn/deployment.py | 5 ++++- .../resnet50_maskrcnn/deployment.py | 5 ++++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/otx/algorithms/detection/adapters/openvino/task.py b/otx/algorithms/detection/adapters/openvino/task.py index 5a823e00c8a..36992c04a73 100644 --- a/otx/algorithms/detection/adapters/openvino/task.py +++ b/otx/algorithms/detection/adapters/openvino/task.py @@ -370,6 +370,7 @@ def load_config(self) -> ADDict: flatten_detection_config_groups(config) try: if self.model is not None and self.model.get_data("config.json"): + # TODO[EUGENE]: ir scale config handling json_dict = json.loads(self.model.get_data("config.json")) flatten_config_values(json_dict) config = merge_a_into_b(json_dict, config) @@ -413,6 +414,7 @@ def load_inferencer( logger.info("Tile classifier is enabled. Load tile classifier model.") tile_classifier_model_file = self.model.get_data("tile_classifier.xml") tile_classifier_weight_file = self.model.get_data("tile_classifier.bin") + # TODO[EUGENE]: ir scale config handling inferencer = OpenVINOTileClassifierWrapper( inferencer, self.config.tiling_parameters.tile_size, diff --git a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml index c305992c177..82b64cb7be2 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml +++ b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml @@ -536,5 +536,24 @@ tiling_parameters: visible_in_ui: true warning: null + # TODO[EUGENE]: ir scale + ir_scale_value: + header: OpenVINO IR Scale Value + description: IR Scale Value + affects_outcome_of: TRAINING + default_value: 1.0 + min_value: 0.0 + max_value: 1.0 + type: FLOAT + editable: true + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 1.0 + visible_in_ui: true + warning: null + type: PARAMETER_GROUP visible_in_ui: true diff --git a/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py b/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py index fa981b687fc..325f3c7954a 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py +++ b/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py @@ -2,11 +2,14 @@ _base_ = ["../../base/deployments/base_instance_segmentation_dynamic.py"] +ir_scale_factor = 2 + ir_config = dict( output_names=["boxes", "labels", "masks"], + input_shape=(1024 * ir_scale_factor, 1024 * ir_scale_factor), ) backend_config = dict( # dynamic batch causes forever running openvino process - model_inputs=[dict(opt_shapes=dict(input=[1, 3, 1024, 1024]))], + model_inputs=[dict(opt_shapes=dict(input=[1, 3, 1024 * ir_scale_factor, 1024 * ir_scale_factor]))], ) diff --git a/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py b/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py index 715e77ef995..7faedef894d 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py +++ b/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py @@ -2,11 +2,14 @@ _base_ = ["../../base/deployments/base_instance_segmentation_dynamic.py"] +ir_scale_factor = 2 + ir_config = dict( output_names=["boxes", "labels", "masks"], + input_shape=(1344 * ir_scale_factor, 800 * ir_scale_factor), ) backend_config = dict( # dynamic batch causes forever running openvino process - model_inputs=[dict(opt_shapes=dict(input=[1, 3, 800, 1344]))], + model_inputs=[dict(opt_shapes=dict(input=[1, 3, 800 * ir_scale_factor, 1344 * ir_scale_factor]))], ) From a7b4acfc59840deeeae62bed0f0fc761792941f9 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 20 Apr 2023 16:04:17 +0100 Subject: [PATCH 2/8] udpate pytest skip tag --- tests/e2e/cli/detection/test_tiling_detection.py | 10 +++++----- .../test_instance_segmentation.py | 10 +++++----- .../cli/instance_segmentation/test_tiling_instseg.py | 10 +++++----- .../integration/cli/detection/test_tiling_detection.py | 2 +- .../cli/instance_segmentation/test_tiling_instseg.py | 2 +- tests/regression/detection/test_tiling_detection.py | 2 +- .../test_tiling_instnace_segmentation.py | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/e2e/cli/detection/test_tiling_detection.py b/tests/e2e/cli/detection/test_tiling_detection.py index f4163bba5c6..0c30964dbc9 100644 --- a/tests/e2e/cli/detection/test_tiling_detection.py +++ b/tests/e2e/cli/detection/test_tiling_detection.py @@ -191,7 +191,7 @@ def test_otx_hpo(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" if template.entrypoints.nncf is None: @@ -202,7 +202,7 @@ def test_nncf_optimize(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_export(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" if template.entrypoints.nncf is None: @@ -213,7 +213,7 @@ def test_nncf_export(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_validate_fq(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" if template.entrypoints.nncf is None: @@ -224,7 +224,7 @@ def test_nncf_validate_fq(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" if template.entrypoints.nncf is None: @@ -235,7 +235,7 @@ def test_nncf_eval(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" if template.entrypoints.nncf is None: diff --git a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py index 0c43092e08a..29bf3855b87 100644 --- a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py +++ b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py @@ -198,7 +198,7 @@ def test_otx_hpo(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "ins_seg" if template.entrypoints.nncf is None: @@ -209,7 +209,7 @@ def test_nncf_optimize(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_export(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "ins_seg" if template.entrypoints.nncf is None: @@ -220,7 +220,7 @@ def test_nncf_export(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_validate_fq(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "ins_seg" if template.entrypoints.nncf is None: @@ -231,7 +231,7 @@ def test_nncf_validate_fq(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "ins_seg" if template.entrypoints.nncf is None: @@ -242,7 +242,7 @@ def test_nncf_eval(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "ins_seg" if template.entrypoints.nncf is None: diff --git a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py index 04bdbb2087f..37ae13ae5fa 100644 --- a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py @@ -192,7 +192,7 @@ def test_otx_hpo(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" if template.entrypoints.nncf is None: @@ -203,7 +203,7 @@ def test_nncf_optimize(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_export(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" if template.entrypoints.nncf is None: @@ -214,7 +214,7 @@ def test_nncf_export(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_validate_fq(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" if template.entrypoints.nncf is None: @@ -225,7 +225,7 @@ def test_nncf_validate_fq(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" if template.entrypoints.nncf is None: @@ -236,7 +236,7 @@ def test_nncf_eval(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_eval_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" if template.entrypoints.nncf is None: diff --git a/tests/integration/cli/detection/test_tiling_detection.py b/tests/integration/cli/detection/test_tiling_detection.py index b4a327839f1..16252c2e91d 100644 --- a/tests/integration/cli/detection/test_tiling_detection.py +++ b/tests/integration/cli/detection/test_tiling_detection.py @@ -112,7 +112,7 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" diff --git a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py index e01acea4af5..a89687f6076 100644 --- a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py @@ -146,7 +146,7 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" diff --git a/tests/regression/detection/test_tiling_detection.py b/tests/regression/detection/test_tiling_detection.py index 04bb571aa8b..7838fe14253 100644 --- a/tests/regression/detection/test_tiling_detection.py +++ b/tests/regression/detection/test_tiling_detection.py @@ -179,7 +179,7 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} diff --git a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py index bc1a9bf5c7d..ac2a004fd94 100644 --- a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py +++ b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py @@ -179,7 +179,7 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-98026") + @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} From 120aebdb1270a333319b907f299f40def62a68a3 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Fri, 21 Apr 2023 17:09:21 +0100 Subject: [PATCH 3/8] add spatial concatenation --- .../common/configs/training_base.py | 16 ++++ .../detection/adapters/mmdet/task.py | 65 +------------ .../adapters/mmdet/utils/__init__.py | 6 ++ .../adapters/mmdet/utils/config_utils.py | 94 +++++++++++++++++++ .../detection/adapters/openvino/task.py | 7 +- .../instance_segmentation/configuration.yaml | 15 ++- .../efficientnetb2b_maskrcnn/deployment.py | 6 +- .../resnet50_maskrcnn/deployment.py | 6 +- otx/api/utils/tiler.py | 2 +- 9 files changed, 139 insertions(+), 78 deletions(-) diff --git a/otx/algorithms/common/configs/training_base.py b/otx/algorithms/common/configs/training_base.py index a57271dc031..0baa0904304 100644 --- a/otx/algorithms/common/configs/training_base.py +++ b/otx/algorithms/common/configs/training_base.py @@ -362,4 +362,20 @@ class BaseTilingParameters(ParameterGroup): affects_outcome_of=ModelLifecycle.NONE, ) + ir_scale_factor = configurable_float( + header="OpenVINO IR Scale Factor", + description="The purpose of the scale parameter is to optimize the performance and " + "efficiency of tiling in OpenVINO IR during inference. By controlling the increase in tile size and " + "input size, the scale parameter allows for more efficient parallelization of the workload and " + "improve the overall performance and efficiency of the inference process on OpenVINO." , + warning="Setting the scale factor value too high may cause the application " + "to crash or result in out-of-memory errors. It is recommended to " + "adjust the scale factor value carefully based on the available " + "hardware resources and the needs of the application.", + default_value=2.0, + min_value=1.0, + max_value=4.0, + affects_outcome_of=ModelLifecycle.NONE, + ) + tiling_parameters = add_parameter_group(BaseTilingParameters) diff --git a/otx/algorithms/detection/adapters/mmdet/task.py b/otx/algorithms/detection/adapters/mmdet/task.py index 3115592ff86..6d8fbdfde37 100644 --- a/otx/algorithms/detection/adapters/mmdet/task.py +++ b/otx/algorithms/detection/adapters/mmdet/task.py @@ -60,7 +60,7 @@ from otx.algorithms.detection.adapters.mmdet.hooks.det_class_probability_map_hook import ( DetClassProbabilityMapHook, ) -from otx.algorithms.detection.adapters.mmdet.utils import patch_tiling +from otx.algorithms.detection.adapters.mmdet.utils import patch_tiling, patch_input_preprocessing, patch_input_shape, patch_ir_scale_factor from otx.algorithms.detection.adapters.mmdet.utils.builder import build_detector from otx.algorithms.detection.adapters.mmdet.utils.config_utils import ( should_cluster_anchors, @@ -616,67 +616,10 @@ def _init_deploy_cfg(self, cfg) -> Union[Config, None]: if os.path.exists(deploy_cfg_path): deploy_cfg = MPAConfig.fromfile(deploy_cfg_path) - def patch_input_preprocessing(deploy_cfg): - normalize_cfg = get_configs_by_pairs( - cfg.data.test.pipeline, - dict(type="Normalize"), - ) - assert len(normalize_cfg) == 1 - normalize_cfg = normalize_cfg[0] - - options = dict(flags=[], args={}) - # NOTE: OTX loads image in RGB format - # so that `to_rgb=True` means a format change to BGR instead. - # Conventionally, OpenVINO IR expects a image in BGR format - # but OpenVINO IR under OTX assumes a image in RGB format. - # - # `to_rgb=True` -> a model was trained with images in BGR format - # and a OpenVINO IR needs to reverse input format from RGB to BGR - # `to_rgb=False` -> a model was trained with images in RGB format - # and a OpenVINO IR does not need to do a reverse - if normalize_cfg.get("to_rgb", False): - options["flags"] += ["--reverse_input_channels"] - # value must be a list not a tuple - if normalize_cfg.get("mean", None) is not None: - options["args"]["--mean_values"] = list(normalize_cfg.get("mean")) - if normalize_cfg.get("std", None) is not None: - options["args"]["--scale_values"] = list(normalize_cfg.get("std")) - - # fill default - backend_config = deploy_cfg.backend_config - if backend_config.get("mo_options") is None: - backend_config.mo_options = ConfigDict() - mo_options = backend_config.mo_options - if mo_options.get("args") is None: - mo_options.args = ConfigDict() - if mo_options.get("flags") is None: - mo_options.flags = [] - - # already defiend options have higher priority - options["args"].update(mo_options.args) - mo_options.args = ConfigDict(options["args"]) - # make sure no duplicates - mo_options.flags.extend(options["flags"]) - mo_options.flags = list(set(mo_options.flags)) - - def patch_input_shape(deploy_cfg): - resize_cfg = get_configs_by_pairs( - cfg.data.test.pipeline, - dict(type="Resize"), - ) - assert len(resize_cfg) == 1 - resize_cfg = resize_cfg[0] - size = resize_cfg.size - if isinstance(size, int): - size = (size, size) - assert all(isinstance(i, int) and i > 0 for i in size) - # default is static shape to prevent an unexpected error - # when converting to OpenVINO IR - deploy_cfg.backend_config.model_inputs = [ConfigDict(opt_shapes=ConfigDict(input=[1, 3, *size]))] - - patch_input_preprocessing(deploy_cfg) + patch_input_preprocessing(cfg, deploy_cfg) if not deploy_cfg.backend_config.get("model_inputs", []): - patch_input_shape(deploy_cfg) + patch_input_shape(cfg, deploy_cfg) + patch_ir_scale_factor(deploy_cfg, self._hyperparams) return deploy_cfg diff --git a/otx/algorithms/detection/adapters/mmdet/utils/__init__.py b/otx/algorithms/detection/adapters/mmdet/utils/__init__.py index 18238cfbb1b..0f65f801642 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/__init__.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/__init__.py @@ -10,6 +10,9 @@ patch_datasets, patch_evaluation, patch_tiling, + patch_input_preprocessing, + patch_input_shape, + patch_ir_scale_factor, prepare_for_training, set_hyperparams, ) @@ -23,4 +26,7 @@ "set_hyperparams", "build_detector", "patch_tiling", + "patch_input_preprocessing", + "patch_input_shape", + "patch_ir_scale_factor", ] diff --git a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py index 16e8ff8f518..bcacec3f999 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py @@ -373,3 +373,97 @@ def patch_tiling(config, hparams, dataset=None): config.update(dict(evaluation=dict(iou_thr=[0.5]))) return config + + +def patch_input_preprocessing(cfg: ConfigDict, deploy_cfg: ConfigDict): + """Update backend configuration with input preprocessing options. + + - If `"to_rgb"` in Normalize config is truthy, it adds `"--reverse_input_channels"` as a flag. + + The function then sets default values for the backend configuration in `deploy_cfg`. + + Args: + cfg (mmcv.ConfigDict): Config object containing test pipeline and other configurations. + deploy_cfg (mmcv.ConfigDict): DeployConfig object containing backend configuration. + + Returns: + None: This function updates the input `deploy_cfg` object directly. + """ + normalize_cfg = get_configs_by_pairs(cfg.data.test.pipeline, dict(type="Normalize")) + assert len(normalize_cfg) == 1 + normalize_cfg = normalize_cfg[0] + + # Set options based on Normalize config + options = { + "flags": ["--reverse_input_channels"] if normalize_cfg.get("to_rgb", False) else [], + "args": { + "--mean_values": list(normalize_cfg.get("mean", [])), + "--scale_values": list(normalize_cfg.get("std", [])) + } + } + + # Set default backend configuration + mo_options = deploy_cfg.backend_config.get("mo_options", ConfigDict()) + mo_options.args = mo_options.get("args", ConfigDict()) + mo_options.flags = mo_options.get("flags", []) + + # Override backend configuration with options from Normalize config + mo_options.args.update(options["args"]) + mo_options.flags = list(set(mo_options.flags + options["flags"])) + + deploy_cfg.backend_config.mo_options = mo_options + + +def patch_input_shape(cfg: ConfigDict, deploy_cfg: ConfigDict): + """Update backend configuration with input shape information. + + This function retrieves the Resize config from `cfg.data.test.pipeline`, checks + that only one Resize then sets the input shape for the backend model in `deploy_cfg` + + ``` + { + "opt_shapes": { + "input": [1, 3, *size] + } + } + ``` + + Args: + cfg (Config): Config object containing test pipeline and other configurations. + deploy_cfg (DeployConfig): DeployConfig object containing backend configuration. + + Returns: + None: This function updates the input `deploy_cfg` object directly. + """ + resize_cfg = get_configs_by_pairs( + cfg.data.test.pipeline, + dict(type="Resize"), + ) + assert len(resize_cfg) == 1 + resize_cfg = resize_cfg[0] + size = resize_cfg.size + if isinstance(size, int): + size = (size, size) + assert all(isinstance(i, int) and i > 0 for i in size) + # default is static shape to prevent an unexpected error + # when converting to OpenVINO IR + deploy_cfg.backend_config.model_inputs = [ConfigDict(opt_shapes=ConfigDict(input=[1, 3, *size]))] + + +def patch_ir_scale_factor(deploy_cfg: ConfigDict, hyper_parameters: DetectionConfig): + """ Patch IR scale factor inplace from hyper parameters to deploy config. + + Args: + deploy_cfg (ConfigDict): mmcv deploy config + hyper_parameters (DetectionConfig): OTX detection hyper parameters + """ + + if hyper_parameters.tiling_parameters.enable_tiling: + scale_ir_input = deploy_cfg.get("scale_ir_input", False) + if scale_ir_input: + ir_scale_factor = hyper_parameters.tiling_parameters.ir_scale_factor + logger.info(f"Apply OpenVINO IR scale factor: {ir_scale_factor}") + ir_input_shape = deploy_cfg.backend_config.model_inputs[0].opt_shapes.input + ir_input_shape[2] = int(ir_input_shape[2] * ir_scale_factor) # height + ir_input_shape[3] = int(ir_input_shape[3] * ir_scale_factor) # width + deploy_cfg.ir_config.input_shape = (ir_input_shape[3], ir_input_shape[2]) # width, height diff --git a/otx/algorithms/detection/adapters/openvino/task.py b/otx/algorithms/detection/adapters/openvino/task.py index 36992c04a73..416f242bc32 100644 --- a/otx/algorithms/detection/adapters/openvino/task.py +++ b/otx/algorithms/detection/adapters/openvino/task.py @@ -274,6 +274,7 @@ def __init__( tile_size: int = 400, overlap: float = 0.5, max_number: int = 100, + ir_scale_factor: float = 1.0, tile_classifier_model_file: Union[str, bytes, None] = None, tile_classifier_weight_file: Union[str, bytes, None] = None, device: str = "CPU", @@ -293,7 +294,7 @@ def __init__( classifier = Model(model_adapter=adapter, preload=True) self.tiler = Tiler( - tile_size=tile_size, + tile_size=int(tile_size * ir_scale_factor), overlap=overlap, max_number=max_number, detector=inferencer.model, @@ -370,9 +371,10 @@ def load_config(self) -> ADDict: flatten_detection_config_groups(config) try: if self.model is not None and self.model.get_data("config.json"): - # TODO[EUGENE]: ir scale config handling json_dict = json.loads(self.model.get_data("config.json")) flatten_config_values(json_dict) + # NOTE: for backward compatibility + json_dict['tiling_parameters']['ir_scale_factor'] = json_dict['tiling_parameters'].get("ir_scale_factor", 1.0) config = merge_a_into_b(json_dict, config) except Exception as e: # pylint: disable=broad-except logger.warning(f"Failed to load config.json: {e}") @@ -420,6 +422,7 @@ def load_inferencer( self.config.tiling_parameters.tile_size, self.config.tiling_parameters.tile_overlap, self.config.tiling_parameters.tile_max_number, + self.config.tiling_parameters.ir_scale_factor, tile_classifier_model_file, tile_classifier_weight_file, ) diff --git a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml index 82b64cb7be2..3ca36484f50 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml +++ b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml @@ -536,14 +536,13 @@ tiling_parameters: visible_in_ui: true warning: null - # TODO[EUGENE]: ir scale - ir_scale_value: - header: OpenVINO IR Scale Value - description: IR Scale Value + ir_scale_factor: + header: OpenVINO IR Scale Factor + description: The purpose of the scale parameter is to optimize the performance and efficiency of tiling in OpenVINO IR during inference. By controlling the increase in tile size and input size, the scale parameter allows for more efficient parallelization of the workload and improve the overall performance and efficiency of the inference process on OpenVINO. affects_outcome_of: TRAINING - default_value: 1.0 - min_value: 0.0 - max_value: 1.0 + default_value: 2.0 + min_value: 1.0 + max_value: 4.0 type: FLOAT editable: true ui_rules: @@ -551,7 +550,7 @@ tiling_parameters: operator: AND rules: [] type: UI_RULES - value: 1.0 + value: 2.0 visible_in_ui: true warning: null diff --git a/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py b/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py index 325f3c7954a..f9701f38ac0 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py +++ b/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/deployment.py @@ -2,14 +2,14 @@ _base_ = ["../../base/deployments/base_instance_segmentation_dynamic.py"] -ir_scale_factor = 2 +scale_ir_input = True ir_config = dict( output_names=["boxes", "labels", "masks"], - input_shape=(1024 * ir_scale_factor, 1024 * ir_scale_factor), + input_shape=(1024, 1024), ) backend_config = dict( # dynamic batch causes forever running openvino process - model_inputs=[dict(opt_shapes=dict(input=[1, 3, 1024 * ir_scale_factor, 1024 * ir_scale_factor]))], + model_inputs=[dict(opt_shapes=dict(input=[1, 3, 1024, 1024]))], ) diff --git a/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py b/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py index 7faedef894d..8ef82f1ca34 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py +++ b/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/deployment.py @@ -2,14 +2,14 @@ _base_ = ["../../base/deployments/base_instance_segmentation_dynamic.py"] -ir_scale_factor = 2 +scale_ir_input = True ir_config = dict( output_names=["boxes", "labels", "masks"], - input_shape=(1344 * ir_scale_factor, 800 * ir_scale_factor), + input_shape=(1344, 800), ) backend_config = dict( # dynamic batch causes forever running openvino process - model_inputs=[dict(opt_shapes=dict(input=[1, 3, 800 * ir_scale_factor, 1344 * ir_scale_factor]))], + model_inputs=[dict(opt_shapes=dict(input=[1, 3, 800, 1344]))], ) diff --git a/otx/api/utils/tiler.py b/otx/api/utils/tiler.py index cbbabae68ae..8f5bba63507 100644 --- a/otx/api/utils/tiler.py +++ b/otx/api/utils/tiler.py @@ -70,7 +70,7 @@ def tile(self, image: np.ndarray) -> List[List[int]]: return coords def filter_tiles_by_objectness( - self, image: np.ndarray, tile_coords: List[List[int]], confidence_threshold: float = 0.45 + self, image: np.ndarray, tile_coords: List[List[int]], confidence_threshold: float = 0.35 ): """Filter tiles by objectness score by running tile classifier. From 4e6bf3d85b6584d438d72d9ce1ee24b34df53873 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Fri, 21 Apr 2023 17:24:32 +0100 Subject: [PATCH 4/8] code quality fixes --- otx/algorithms/common/configs/training_base.py | 2 +- .../detection/adapters/mmdet/task.py | 8 ++++++-- .../detection/adapters/mmdet/utils/__init__.py | 2 +- .../adapters/mmdet/utils/config_utils.py | 18 +++++++++--------- .../detection/adapters/openvino/task.py | 4 +++- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/otx/algorithms/common/configs/training_base.py b/otx/algorithms/common/configs/training_base.py index 0baa0904304..202e405ccc3 100644 --- a/otx/algorithms/common/configs/training_base.py +++ b/otx/algorithms/common/configs/training_base.py @@ -367,7 +367,7 @@ class BaseTilingParameters(ParameterGroup): description="The purpose of the scale parameter is to optimize the performance and " "efficiency of tiling in OpenVINO IR during inference. By controlling the increase in tile size and " "input size, the scale parameter allows for more efficient parallelization of the workload and " - "improve the overall performance and efficiency of the inference process on OpenVINO." , + "improve the overall performance and efficiency of the inference process on OpenVINO.", warning="Setting the scale factor value too high may cause the application " "to crash or result in out-of-memory errors. It is recommended to " "adjust the scale factor value carefully based on the available " diff --git a/otx/algorithms/detection/adapters/mmdet/task.py b/otx/algorithms/detection/adapters/mmdet/task.py index 6d8fbdfde37..31fce66e0f1 100644 --- a/otx/algorithms/detection/adapters/mmdet/task.py +++ b/otx/algorithms/detection/adapters/mmdet/task.py @@ -39,7 +39,6 @@ ) from otx.algorithms.common.adapters.mmcv.utils import ( build_data_parallel, - get_configs_by_pairs, patch_data_pipeline, patch_from_hyperparams, ) @@ -60,7 +59,12 @@ from otx.algorithms.detection.adapters.mmdet.hooks.det_class_probability_map_hook import ( DetClassProbabilityMapHook, ) -from otx.algorithms.detection.adapters.mmdet.utils import patch_tiling, patch_input_preprocessing, patch_input_shape, patch_ir_scale_factor +from otx.algorithms.detection.adapters.mmdet.utils import ( + patch_input_preprocessing, + patch_input_shape, + patch_ir_scale_factor, + patch_tiling, +) from otx.algorithms.detection.adapters.mmdet.utils.builder import build_detector from otx.algorithms.detection.adapters.mmdet.utils.config_utils import ( should_cluster_anchors, diff --git a/otx/algorithms/detection/adapters/mmdet/utils/__init__.py b/otx/algorithms/detection/adapters/mmdet/utils/__init__.py index 0f65f801642..77b125b0ca5 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/__init__.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/__init__.py @@ -9,10 +9,10 @@ patch_config, patch_datasets, patch_evaluation, - patch_tiling, patch_input_preprocessing, patch_input_shape, patch_ir_scale_factor, + patch_tiling, prepare_for_training, set_hyperparams, ) diff --git a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py index bcacec3f999..9d99ff531f7 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py @@ -389,17 +389,17 @@ def patch_input_preprocessing(cfg: ConfigDict, deploy_cfg: ConfigDict): Returns: None: This function updates the input `deploy_cfg` object directly. """ - normalize_cfg = get_configs_by_pairs(cfg.data.test.pipeline, dict(type="Normalize")) - assert len(normalize_cfg) == 1 - normalize_cfg = normalize_cfg[0] + normalize_cfgs = get_configs_by_pairs(cfg.data.test.pipeline, dict(type="Normalize")) + assert len(normalize_cfgs) == 1 + normalize_cfg: dict = normalize_cfgs[0] # Set options based on Normalize config options = { "flags": ["--reverse_input_channels"] if normalize_cfg.get("to_rgb", False) else [], "args": { "--mean_values": list(normalize_cfg.get("mean", [])), - "--scale_values": list(normalize_cfg.get("std", [])) - } + "--scale_values": list(normalize_cfg.get("std", [])), + }, } # Set default backend configuration @@ -435,12 +435,12 @@ def patch_input_shape(cfg: ConfigDict, deploy_cfg: ConfigDict): Returns: None: This function updates the input `deploy_cfg` object directly. """ - resize_cfg = get_configs_by_pairs( + resize_cfgs = get_configs_by_pairs( cfg.data.test.pipeline, dict(type="Resize"), ) - assert len(resize_cfg) == 1 - resize_cfg = resize_cfg[0] + assert len(resize_cfgs) == 1 + resize_cfg: ConfigDict = resize_cfgs[0] size = resize_cfg.size if isinstance(size, int): size = (size, size) @@ -451,7 +451,7 @@ def patch_input_shape(cfg: ConfigDict, deploy_cfg: ConfigDict): def patch_ir_scale_factor(deploy_cfg: ConfigDict, hyper_parameters: DetectionConfig): - """ Patch IR scale factor inplace from hyper parameters to deploy config. + """Patch IR scale factor inplace from hyper parameters to deploy config. Args: deploy_cfg (ConfigDict): mmcv deploy config diff --git a/otx/algorithms/detection/adapters/openvino/task.py b/otx/algorithms/detection/adapters/openvino/task.py index 416f242bc32..8d26103edf8 100644 --- a/otx/algorithms/detection/adapters/openvino/task.py +++ b/otx/algorithms/detection/adapters/openvino/task.py @@ -374,7 +374,9 @@ def load_config(self) -> ADDict: json_dict = json.loads(self.model.get_data("config.json")) flatten_config_values(json_dict) # NOTE: for backward compatibility - json_dict['tiling_parameters']['ir_scale_factor'] = json_dict['tiling_parameters'].get("ir_scale_factor", 1.0) + json_dict["tiling_parameters"]["ir_scale_factor"] = json_dict["tiling_parameters"].get( + "ir_scale_factor", 1.0 + ) config = merge_a_into_b(json_dict, config) except Exception as e: # pylint: disable=broad-except logger.warning(f"Failed to load config.json: {e}") From ce2d2cfac4c38fc29fc23d267f80ce4e842f47c6 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Mon, 24 Apr 2023 15:14:50 +0100 Subject: [PATCH 5/8] fix tests --- .../detection/adapters/mmdet/utils/config_utils.py | 1 + .../cli/detection/test_tiling_detection.py | 1 - .../instance_segmentation/test_tiling_instseg.py | 1 - .../regression/detection/test_tiling_detection.py | 1 - .../test_tiling_instnace_segmentation.py | 1 - .../detection/tiling/test_tiling_detection.py | 14 +++++++------- 6 files changed, 8 insertions(+), 11 deletions(-) diff --git a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py index 9d99ff531f7..21ff798c35f 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py @@ -404,6 +404,7 @@ def patch_input_preprocessing(cfg: ConfigDict, deploy_cfg: ConfigDict): # Set default backend configuration mo_options = deploy_cfg.backend_config.get("mo_options", ConfigDict()) + mo_options = ConfigDict() if mo_options is None else mo_options mo_options.args = mo_options.get("args", ConfigDict()) mo_options.flags = mo_options.get("flags", []) diff --git a/tests/integration/cli/detection/test_tiling_detection.py b/tests/integration/cli/detection/test_tiling_detection.py index 16252c2e91d..8b27b856e56 100644 --- a/tests/integration/cli/detection/test_tiling_detection.py +++ b/tests/integration/cli/detection/test_tiling_detection.py @@ -112,7 +112,6 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.skip(reason="CVS-109001") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" diff --git a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py index a89687f6076..9f5f466f6f3 100644 --- a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py @@ -146,7 +146,6 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.skip(reason="CVS-109001") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" diff --git a/tests/regression/detection/test_tiling_detection.py b/tests/regression/detection/test_tiling_detection.py index 7838fe14253..6dc9f62acb4 100644 --- a/tests/regression/detection/test_tiling_detection.py +++ b/tests/regression/detection/test_tiling_detection.py @@ -179,7 +179,6 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} diff --git a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py index ac2a004fd94..dc3651ddc07 100644 --- a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py +++ b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py @@ -179,7 +179,6 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="CVS-109001") def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} diff --git a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py index 70f7a8db998..5594a2cc62a 100644 --- a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py +++ b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py @@ -33,7 +33,7 @@ def create_otx_dataset(height: int, width: int, labels: List[str]): - """Create a random OTX dataset + """Create a random OTX dataset. Args: height (int): The height of the image @@ -54,11 +54,11 @@ def create_otx_dataset(height: int, width: int, labels: List[str]): class TestTilingDetection: - """Test the tiling functionality""" + """Test the tiling detection algorithm.""" @pytest.fixture(autouse=True) def setUp(self) -> None: - """Setup the test case""" + """Setup the test case.""" self.height = 1024 self.width = 1024 self.label_names = ["rectangle", "ellipse", "triangle"] @@ -132,7 +132,7 @@ def setUp(self) -> None: @e2e_pytest_unit def test_tiling_train_dataloader(self): - """Test that the training dataloader is built correctly for tiling""" + """Test that the training dataloader is built correctly for tiling.""" dataset = build_dataset(self.train_data_cfg) train_dataloader = build_dataloader(dataset, **self.dataloader_cfg) @@ -143,7 +143,7 @@ def test_tiling_train_dataloader(self): @e2e_pytest_unit def test_tiling_test_dataloader(self): - """Test that the testing dataloader is built correctly for tiling""" + """Test that the testing dataloader is built correctly for tiling.""" dataset = build_dataset(self.test_data_cfg) stride = int((1 - self.tile_cfg["overlap_ratio"]) * self.tile_cfg["tile_size"]) @@ -160,7 +160,7 @@ def test_tiling_test_dataloader(self): @e2e_pytest_unit def test_inference_merge(self): - """Test that the inference merge works correctly""" + """Test that the inference merge works correctly.""" dataset = build_dataset(self.test_data_cfg) # create simulated inference results @@ -222,7 +222,7 @@ def test_load_tiling_parameters(self, tmp_dir_path): @e2e_pytest_unit def test_patch_tiling_func(self): - """Test that patch_tiling function works correctly""" + """Test that patch_tiling function works correctly.""" cfg = MPAConfig.fromfile(os.path.join(DEFAULT_ISEG_TEMPLATE_DIR, "model.py")) model_template = parse_model_template(os.path.join(DEFAULT_ISEG_TEMPLATE_DIR, "template.yaml")) hyper_parameters = create(model_template.hyper_parameters.data) From 2c129b742f6fe618ca653e4465ac82e29af912e0 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Tue, 25 Apr 2023 19:30:18 +0100 Subject: [PATCH 6/8] add unit test for ir scale factor --- .../detection/adapters/openvino/task.py | 1 - .../detection/tiling/test_tiling_detection.py | 84 ++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/otx/algorithms/detection/adapters/openvino/task.py b/otx/algorithms/detection/adapters/openvino/task.py index 01bf07ee04b..f24828beb76 100644 --- a/otx/algorithms/detection/adapters/openvino/task.py +++ b/otx/algorithms/detection/adapters/openvino/task.py @@ -418,7 +418,6 @@ def load_inferencer( logger.info("Tile classifier is enabled. Load tile classifier model.") tile_classifier_model_file = self.model.get_data("tile_classifier.xml") tile_classifier_weight_file = self.model.get_data("tile_classifier.bin") - # TODO[EUGENE]: ir scale config handling inferencer = OpenVINOTileClassifierWrapper( inferencer, self.config.tiling_parameters.tile_size, diff --git a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py index 5594a2cc62a..3cc27a3d72d 100644 --- a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py +++ b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py @@ -8,10 +8,14 @@ import numpy as np import pytest import torch -from mmcv import ConfigDict +from mmcv import Config, ConfigDict from mmdet.datasets import build_dataloader, build_dataset +from mmdet.models import DETECTORS +from openvino.model_zoo.model_api.adapters import OpenvinoAdapter, create_core +from torch import nn from otx.algorithms.common.adapters.mmcv.utils.config_utils import MPAConfig +from otx.algorithms.common.adapters.mmdeploy.apis import MMdeployExporter from otx.algorithms.detection.adapters.mmdet.task import MMDetectionTask from otx.algorithms.detection.adapters.mmdet.utils import build_detector, patch_tiling from otx.api.configuration.helper import create @@ -32,6 +36,24 @@ ) +@DETECTORS.register_module(force=True) +class MockDetModel(nn.Module): + def __init__(self, backbone, train_cfg=None, test_cfg=None, init_cfg=None): + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + self.box_dummy = torch.nn.AdaptiveAvgPool2d((1, 5)) + self.label_dummy = torch.nn.AdaptiveAvgPool2d((1)) + self.mask_dummy = torch.nn.AdaptiveAvgPool2d((28, 28)) + + def forward(self, *args, **kwargs): + img = args[0] + x = self.conv(img) + boxes = self.box_dummy(x).mean(1) + labels = self.label_dummy(x).mean(1) + masks = self.mask_dummy(x).mean(1) + return boxes, labels, masks + + def create_otx_dataset(height: int, width: int, labels: List[str]): """Create a random OTX dataset. @@ -235,6 +257,60 @@ def test_patch_tiling_func(self): patch_tiling(cfg, hyper_parameters, self.otx_dataset) @e2e_pytest_unit - def test_openvino(self): - # TODO[EUGENE]: implement unittest for tiling prediction with openvino - pass + @pytest.mark.parametrize("scale_factor", [1, 1.5, 2, 3, 4]) + def test_ir_scale_deploy(self, tmp_dir_path, scale_factor): + """Test that the IR scale factor is correctly applied during inference.""" + model_template = parse_model_template(os.path.join(DEFAULT_ISEG_TEMPLATE_DIR, "template.yaml")) + hyper_parameters = create(model_template.hyper_parameters.data) + hyper_parameters.tiling_parameters.enable_tiling = True + hyper_parameters.tiling_parameters.ir_scale_factor = scale_factor + task_env = init_environment(hyper_parameters, model_template) + img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + task = MMDetectionTask(task_env) + pipeline = [ + dict(type="LoadImageFromFile"), + dict( + type="MultiScaleFlipAug", + img_scale=(800, 800), + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=False), + dict(type="RandomFlip"), + dict(type="Normalize", **img_norm_cfg), + dict(type="Pad", size_divisor=32), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img"]), + ], + ), + ] + config = Config( + dict(model=dict(type="MockDetModel", backbone=dict(init_cfg=None)), data=dict(test=dict(pipeline=pipeline))) + ) + + deploy_cfg = task._init_deploy_cfg(config) + onnx_path = MMdeployExporter.torch2onnx( + tmp_dir_path, + np.zeros((50, 50, 3), dtype=np.float32), + config, + deploy_cfg, + ) + assert isinstance(onnx_path, str) + assert os.path.exists(onnx_path) + + openvino_paths = MMdeployExporter.onnx2openvino( + tmp_dir_path, + onnx_path, + deploy_cfg, + ) + for openvino_path in openvino_paths: + assert os.path.exists(openvino_path) + + task._init_task() + original_width, original_height = task._recipe_cfg.data.test.pipeline[0].img_scale # w, h + + model_adapter = OpenvinoAdapter(create_core(), openvino_paths[0], openvino_paths[1]) + + ir_input_shape = model_adapter.get_input_layers()["image"].shape + _, _, ir_height, ir_width = ir_input_shape + assert ir_height == original_height * scale_factor + assert ir_width == original_width * scale_factor From f3bd4c383dd8cbd7dbe41ba238b549ef17bfa7b5 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Wed, 26 Apr 2023 14:10:29 +0100 Subject: [PATCH 7/8] revert pytest skip and address test failure in another PR --- tests/integration/cli/detection/test_tiling_detection.py | 1 + .../integration/cli/instance_segmentation/test_tiling_instseg.py | 1 + tests/regression/detection/test_tiling_detection.py | 1 + .../instance_segmentation/test_tiling_instnace_segmentation.py | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/integration/cli/detection/test_tiling_detection.py b/tests/integration/cli/detection/test_tiling_detection.py index 8b27b856e56..b4a327839f1 100644 --- a/tests/integration/cli/detection/test_tiling_detection.py +++ b/tests/integration/cli/detection/test_tiling_detection.py @@ -112,6 +112,7 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component + @pytest.mark.skip(reason="CVS-98026") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_det" diff --git a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py index 9f5f466f6f3..e01acea4af5 100644 --- a/tests/integration/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/integration/cli/instance_segmentation/test_tiling_instseg.py @@ -146,6 +146,7 @@ def test_otx_hpo(self, template, tmp_dir_path): otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component + @pytest.mark.skip(reason="CVS-98026") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" diff --git a/tests/regression/detection/test_tiling_detection.py b/tests/regression/detection/test_tiling_detection.py index 6dc9f62acb4..d33845c1b58 100644 --- a/tests/regression/detection/test_tiling_detection.py +++ b/tests/regression/detection/test_tiling_detection.py @@ -178,6 +178,7 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): assert test_result["passed"] is True, test_result["log"] @e2e_pytest_component + @pytest.mark.skip(reason="CVS-98026") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} diff --git a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py index dc3651ddc07..61d5d020137 100644 --- a/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py +++ b/tests/regression/instance_segmentation/test_tiling_instnace_segmentation.py @@ -178,6 +178,7 @@ def test_otx_deploy_eval_deployment(self, template, tmp_dir_path): assert test_result["passed"] is True, test_result["log"] @e2e_pytest_component + @pytest.mark.skip(reason="CVS-98026") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_nncf_optimize_eval(self, template, tmp_dir_path): self.performance[template.name] = {} From ce684d2fbc211acdfe375325f21510a571c675b7 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Wed, 26 Apr 2023 17:24:39 +0100 Subject: [PATCH 8/8] refactor variable name --- otx/algorithms/common/configs/training_base.py | 2 +- .../detection/adapters/mmdet/utils/config_utils.py | 8 ++++---- otx/algorithms/detection/adapters/openvino/task.py | 11 ++++++----- .../configs/instance_segmentation/configuration.yaml | 2 +- .../detection/tiling/test_tiling_detection.py | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/otx/algorithms/common/configs/training_base.py b/otx/algorithms/common/configs/training_base.py index 9c67e78f8be..c644c014106 100644 --- a/otx/algorithms/common/configs/training_base.py +++ b/otx/algorithms/common/configs/training_base.py @@ -372,7 +372,7 @@ class BaseTilingParameters(ParameterGroup): affects_outcome_of=ModelLifecycle.NONE, ) - ir_scale_factor = configurable_float( + tile_ir_scale_factor = configurable_float( header="OpenVINO IR Scale Factor", description="The purpose of the scale parameter is to optimize the performance and " "efficiency of tiling in OpenVINO IR during inference. By controlling the increase in tile size and " diff --git a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py index 21ff798c35f..fe3e77c3592 100644 --- a/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py +++ b/otx/algorithms/detection/adapters/mmdet/utils/config_utils.py @@ -462,9 +462,9 @@ def patch_ir_scale_factor(deploy_cfg: ConfigDict, hyper_parameters: DetectionCon if hyper_parameters.tiling_parameters.enable_tiling: scale_ir_input = deploy_cfg.get("scale_ir_input", False) if scale_ir_input: - ir_scale_factor = hyper_parameters.tiling_parameters.ir_scale_factor - logger.info(f"Apply OpenVINO IR scale factor: {ir_scale_factor}") + tile_ir_scale_factor = hyper_parameters.tiling_parameters.tile_ir_scale_factor + logger.info(f"Apply OpenVINO IR scale factor: {tile_ir_scale_factor}") ir_input_shape = deploy_cfg.backend_config.model_inputs[0].opt_shapes.input - ir_input_shape[2] = int(ir_input_shape[2] * ir_scale_factor) # height - ir_input_shape[3] = int(ir_input_shape[3] * ir_scale_factor) # width + ir_input_shape[2] = int(ir_input_shape[2] * tile_ir_scale_factor) # height + ir_input_shape[3] = int(ir_input_shape[3] * tile_ir_scale_factor) # width deploy_cfg.ir_config.input_shape = (ir_input_shape[3], ir_input_shape[2]) # width, height diff --git a/otx/algorithms/detection/adapters/openvino/task.py b/otx/algorithms/detection/adapters/openvino/task.py index f24828beb76..659112dd1d2 100644 --- a/otx/algorithms/detection/adapters/openvino/task.py +++ b/otx/algorithms/detection/adapters/openvino/task.py @@ -261,6 +261,7 @@ class OpenVINOTileClassifierWrapper(BaseInferencerWithConverter): tile_size (int): tile size overlap (float): overlap ratio between tiles max_number (int): maximum number of objects per image + tile_ir_scale_factor (float, optional): scale factor for tile size tile_classifier_model_file (Union[str, bytes, None], optional): tile classifier xml. Defaults to None. tile_classifier_weight_file (Union[str, bytes, None], optional): til classifier weight bin. Defaults to None. device (str, optional): device to run inference on, such as CPU, GPU or MYRIAD. Defaults to "CPU". @@ -274,7 +275,7 @@ def __init__( tile_size: int = 400, overlap: float = 0.5, max_number: int = 100, - ir_scale_factor: float = 1.0, + tile_ir_scale_factor: float = 1.0, tile_classifier_model_file: Union[str, bytes, None] = None, tile_classifier_weight_file: Union[str, bytes, None] = None, device: str = "CPU", @@ -294,7 +295,7 @@ def __init__( classifier = Model(model_adapter=adapter, preload=True) self.tiler = Tiler( - tile_size=int(tile_size * ir_scale_factor), + tile_size=int(tile_size * tile_ir_scale_factor), overlap=overlap, max_number=max_number, detector=inferencer.model, @@ -374,8 +375,8 @@ def load_config(self) -> ADDict: json_dict = json.loads(self.model.get_data("config.json")) flatten_config_values(json_dict) # NOTE: for backward compatibility - json_dict["tiling_parameters"]["ir_scale_factor"] = json_dict["tiling_parameters"].get( - "ir_scale_factor", 1.0 + json_dict["tiling_parameters"]["tile_ir_scale_factor"] = json_dict["tiling_parameters"].get( + "tile_ir_scale_factor", 1.0 ) config = merge_a_into_b(json_dict, config) except Exception as e: # pylint: disable=broad-except @@ -423,7 +424,7 @@ def load_inferencer( self.config.tiling_parameters.tile_size, self.config.tiling_parameters.tile_overlap, self.config.tiling_parameters.tile_max_number, - self.config.tiling_parameters.ir_scale_factor, + self.config.tiling_parameters.tile_ir_scale_factor, tile_classifier_model_file, tile_classifier_weight_file, ) diff --git a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml index 70499acc536..088fb58b33a 100644 --- a/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml +++ b/otx/algorithms/detection/configs/instance_segmentation/configuration.yaml @@ -553,7 +553,7 @@ tiling_parameters: visible_in_ui: true warning: null - ir_scale_factor: + tile_ir_scale_factor: header: OpenVINO IR Scale Factor description: The purpose of the scale parameter is to optimize the performance and efficiency of tiling in OpenVINO IR during inference. By controlling the increase in tile size and input size, the scale parameter allows for more efficient parallelization of the workload and improve the overall performance and efficiency of the inference process on OpenVINO. affects_outcome_of: TRAINING diff --git a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py index 3cc27a3d72d..c1462e8e05b 100644 --- a/tests/unit/algorithms/detection/tiling/test_tiling_detection.py +++ b/tests/unit/algorithms/detection/tiling/test_tiling_detection.py @@ -258,12 +258,12 @@ def test_patch_tiling_func(self): @e2e_pytest_unit @pytest.mark.parametrize("scale_factor", [1, 1.5, 2, 3, 4]) - def test_ir_scale_deploy(self, tmp_dir_path, scale_factor): + def test_tile_ir_scale_deploy(self, tmp_dir_path, scale_factor): """Test that the IR scale factor is correctly applied during inference.""" model_template = parse_model_template(os.path.join(DEFAULT_ISEG_TEMPLATE_DIR, "template.yaml")) hyper_parameters = create(model_template.hyper_parameters.data) hyper_parameters.tiling_parameters.enable_tiling = True - hyper_parameters.tiling_parameters.ir_scale_factor = scale_factor + hyper_parameters.tiling_parameters.tile_ir_scale_factor = scale_factor task_env = init_environment(hyper_parameters, model_template) img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) task = MMDetectionTask(task_env)