diff --git a/tests/openvino/native/common.py b/tests/openvino/native/common.py index c61d56bdbc2..8e4fa7673c8 100644 --- a/tests/openvino/native/common.py +++ b/tests/openvino/native/common.py @@ -22,6 +22,7 @@ from nncf.openvino.graph.nncf_graph_builder import GraphConverter from tests.openvino.conftest import OPENVINO_NATIVE_TEST_ROOT from tests.shared.nx_graph import compare_nx_graph_with_reference +from tests.shared.openvino_version import get_openvino_version def convert_torch_model(model: torch.nn.Module, input_shape: Tuple[int], tmp_path: Path) -> ov.Model: @@ -77,21 +78,6 @@ def dump_to_json(local_path, data): json.dump(deepcopy(data), file, indent=4, cls=NumpyEncoder) -def get_openvino_major_minor_version() -> Tuple[int]: - ov_version = ov.__version__ - pos = ov_version.find("-") - if pos != -1: - ov_version = ov_version[:pos] - - ov_version = version.parse(ov_version).base_version - return tuple(map(int, ov_version.split(".")[:2])) - - -def get_openvino_version() -> str: - major_version, minor_version = get_openvino_major_minor_version() - return f"{major_version}.{minor_version}" - - def get_actual_reference_for_current_openvino(rel_path: Path) -> Path: """ Get path to actual reference file. diff --git a/tests/post_training/test_quantize_conformance.py b/tests/post_training/test_quantize_conformance.py index e0f7a107627..368664de3da 100644 --- a/tests/post_training/test_quantize_conformance.py +++ b/tests/post_training/test_quantize_conformance.py @@ -23,12 +23,12 @@ from packaging import version import nncf -from tests.openvino.native.common import get_openvino_version from tests.post_training.model_scope import PTQ_TEST_CASES from tests.post_training.model_scope import WC_TEST_CASES from tests.post_training.pipelines.base import BackendType from tests.post_training.pipelines.base import BaseTestPipeline from tests.post_training.pipelines.base import RunInfo +from tests.shared.openvino_version import get_openvino_version DATA_ROOT = Path(__file__).parent / "data" diff --git a/tests/shared/openvino_version.py b/tests/shared/openvino_version.py new file mode 100644 index 00000000000..b68c9b8a035 --- /dev/null +++ b/tests/shared/openvino_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple + +import openvino as ov +from packaging import version + + +def get_openvino_major_minor_version() -> Tuple[int]: + ov_version = ov.__version__ + pos = ov_version.find("-") + if pos != -1: + ov_version = ov_version[:pos] + + ov_version = version.parse(ov_version).base_version + return tuple(map(int, ov_version.split(".")[:2])) + + +def get_openvino_version() -> str: + major_version, minor_version = get_openvino_major_minor_version() + return f"{major_version}.{minor_version}" diff --git a/tests/tensorflow/sota_checkpoints_eval.json b/tests/tensorflow/sota_checkpoints_eval.json index 440a713a4ea..4d5c11b11f1 100644 --- a/tests/tensorflow/sota_checkpoints_eval.json +++ b/tests/tensorflow/sota_checkpoints_eval.json @@ -35,7 +35,9 @@ "model_description": "Inception V3", "compression_description": "INT8 (per-tensor symmetric for weights, per-tensor asymmetric half-range for activations), Sparsity 61% (RB)", "target_tf": 77.52, - "target_ov": 77.51 + "target_ov": 77.51, + "skip_ov": "CVS-147730 only for OV 2024.3", + "skip_ov_version": "2024.3" }, "inception_v3_imagenet_magnitude_sparsity": { "config": "examples/tensorflow/classification/configs/sparsity/inception_v3_imagenet_magnitude_sparsity.json", @@ -395,4 +397,4 @@ } } } -} +} \ No newline at end of file diff --git a/tests/tensorflow/test_sota_checkpoints.py b/tests/tensorflow/test_sota_checkpoints.py index 27f9318d15d..c9f07ee1cf4 100644 --- a/tests/tensorflow/test_sota_checkpoints.py +++ b/tests/tensorflow/test_sota_checkpoints.py @@ -28,6 +28,7 @@ from tests.shared.command import Command from tests.shared.metric_thresholds import DIFF_FP32_MAX_GLOBAL from tests.shared.metric_thresholds import DIFF_FP32_MIN_GLOBAL +from tests.shared.openvino_version import get_openvino_version from tests.shared.paths import DATASET_DEFINITIONS_PATH from tests.shared.paths import PROJECT_ROOT from tests.shared.paths import TEST_ROOT @@ -85,6 +86,7 @@ class EvalRunParamsStruct: diff_target_ov_min: float diff_target_ov_max: float skip_ov: Optional[str] + skip_ov_version: Optional[str] xfail_ov: Optional[str] @@ -163,6 +165,7 @@ def read_reference_file(ref_path: Path) -> List[EvalRunParamsStruct]: diff_target_tf_min=sample_dict.get("diff_target_tf_min", DIFF_TARGET_TF_MIN), diff_target_tf_max=sample_dict.get("diff_target_tf_max", DIFF_TARGET_TF_MAX), skip_ov=sample_dict.get("skip_ov"), + skip_ov_version=sample_dict.get("skip_ov_version"), xfail_ov=sample_dict.get("xfail_ov"), ) ) @@ -473,7 +476,9 @@ def test_openvino_eval( ): if not openvino: pytest.skip() - if eval_test_struct.skip_ov: + if eval_test_struct.skip_ov and ( + eval_test_struct.skip_ov_version is None or eval_test_struct.skip_ov_version == get_openvino_version() + ): status = f"Skip by: {eval_test_struct.skip_ov}" collected_data.append(ResultInfo(model_name=eval_test_struct.model_name, backend="OV", status=status)) pytest.skip(status)