Merge branch 'openvinotoolkit:develop' into develop

tests/openvino/native/common.py

@@ -22,6 +22,7 @@
 from nncf.openvino.graph.nncf_graph_builder import GraphConverter
 from tests.openvino.conftest import OPENVINO_NATIVE_TEST_ROOT
 from tests.shared.nx_graph import compare_nx_graph_with_reference
+from tests.shared.openvino_version import get_openvino_version
 
 
 def convert_torch_model(model: torch.nn.Module, input_shape: Tuple[int], tmp_path: Path) -> ov.Model:
@@ -77,21 +78,6 @@ def dump_to_json(local_path, data):
         json.dump(deepcopy(data), file, indent=4, cls=NumpyEncoder)
 
 
-def get_openvino_major_minor_version() -> Tuple[int]:
-    ov_version = ov.__version__
-    pos = ov_version.find("-")
-    if pos != -1:
-        ov_version = ov_version[:pos]
-
-    ov_version = version.parse(ov_version).base_version
-    return tuple(map(int, ov_version.split(".")[:2]))
-
-
-def get_openvino_version() -> str:
-    major_version, minor_version = get_openvino_major_minor_version()
-    return f"{major_version}.{minor_version}"
-
-
 def get_actual_reference_for_current_openvino(rel_path: Path) -> Path:
     """
     Get path to actual reference file.

tests/post_training/test_quantize_conformance.py

@@ -23,12 +23,12 @@
 from packaging import version
 
 import nncf
-from tests.openvino.native.common import get_openvino_version
 from tests.post_training.model_scope import PTQ_TEST_CASES
 from tests.post_training.model_scope import WC_TEST_CASES
 from tests.post_training.pipelines.base import BackendType
 from tests.post_training.pipelines.base import BaseTestPipeline
 from tests.post_training.pipelines.base import RunInfo
+from tests.shared.openvino_version import get_openvino_version
 
 DATA_ROOT = Path(__file__).parent / "data"
 

tests/shared/openvino_version.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2024 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple
+
+import openvino as ov
+from packaging import version
+
+
+def get_openvino_major_minor_version() -> Tuple[int]:
+    ov_version = ov.__version__
+    pos = ov_version.find("-")
+    if pos != -1:
+        ov_version = ov_version[:pos]
+
+    ov_version = version.parse(ov_version).base_version
+    return tuple(map(int, ov_version.split(".")[:2]))
+
+
+def get_openvino_version() -> str:
+    major_version, minor_version = get_openvino_major_minor_version()
+    return f"{major_version}.{minor_version}"

tests/tensorflow/sota_checkpoints_eval.json

@@ -35,7 +35,9 @@
                     "model_description": "Inception V3",
                     "compression_description": "INT8 (per-tensor symmetric for weights, per-tensor asymmetric half-range for activations), Sparsity 61% (RB)",
                     "target_tf": 77.52,
-                    "target_ov": 77.51
+                    "target_ov": 77.51,
+                    "skip_ov": "CVS-147730 only for OV 2024.3",
+                    "skip_ov_version": "2024.3"
                 },
                 "inception_v3_imagenet_magnitude_sparsity": {
                     "config": "examples/tensorflow/classification/configs/sparsity/inception_v3_imagenet_magnitude_sparsity.json",
@@ -395,4 +397,4 @@
             }
         }
     }
-}
+}

tests/tensorflow/test_sota_checkpoints.py

@@ -28,6 +28,7 @@
 from tests.shared.command import Command
 from tests.shared.metric_thresholds import DIFF_FP32_MAX_GLOBAL
 from tests.shared.metric_thresholds import DIFF_FP32_MIN_GLOBAL
+from tests.shared.openvino_version import get_openvino_version
 from tests.shared.paths import DATASET_DEFINITIONS_PATH
 from tests.shared.paths import PROJECT_ROOT
 from tests.shared.paths import TEST_ROOT
@@ -85,6 +86,7 @@ class EvalRunParamsStruct:
     diff_target_ov_min: float
     diff_target_ov_max: float
     skip_ov: Optional[str]
+    skip_ov_version: Optional[str]
     xfail_ov: Optional[str]
 
 
@@ -163,6 +165,7 @@ def read_reference_file(ref_path: Path) -> List[EvalRunParamsStruct]:
                         diff_target_tf_min=sample_dict.get("diff_target_tf_min", DIFF_TARGET_TF_MIN),
                         diff_target_tf_max=sample_dict.get("diff_target_tf_max", DIFF_TARGET_TF_MAX),
                         skip_ov=sample_dict.get("skip_ov"),
+                        skip_ov_version=sample_dict.get("skip_ov_version"),
                         xfail_ov=sample_dict.get("xfail_ov"),
                     )
                 )
@@ -473,7 +476,9 @@ def test_openvino_eval(
     ):
         if not openvino:
             pytest.skip()
-        if eval_test_struct.skip_ov:
+        if eval_test_struct.skip_ov and (
+            eval_test_struct.skip_ov_version is None or eval_test_struct.skip_ov_version == get_openvino_version()
+        ):
             status = f"Skip by: {eval_test_struct.skip_ov}"
             collected_data.append(ResultInfo(model_name=eval_test_struct.model_name, backend="OV", status=status))
             pytest.skip(status)