Use convert_model in tests (#2102)

### Changes Use mo.convert_model instead of converting via onnx in tests. - Update __version__ to 2.6.0, mo.convert_model contains check `nncf.__version__ >= 2.6.0` - Add dump fp32 models - Disable POT backend for `inception_resnet_v2, levit_128, vgg11` in test_quantize_conformance.py (120012) - Update metrics of test_quantize_conformance.py | Model | 2023.0.1 | 2023.1.0 | Diff | |-----------------------------------------------------|------------|------------|---------| | timm/efficientnet_b0_backend_OLD_TORCH | 0.74976 | 0.75538 | 0.0056 | | timm/efficientnet_b0_BC_backend_POT | 0.76878 | 0.76872 | -0.0001 | | timm/levit_128_backend_ONNX | 0.73082 | 0.73184 | 0.0010 | | timm/mobilenetv2_050_backend_POT | 0.64234 | 0.64362 | 0.0013 | | timm/mobilenetv2_050_BC_backend_POT | 0.64376 | 0.6431 | -0.0007 | | timm/mobilenetv3_small_050_backend_POT | 0.27062 | 0.54104 | 0.2704 | | timm/swin_base_patch4_window7_224_backend_OLD_TORCH | 0.79586 | 0.81376 | 0.0179 | | timm/swin_base_patch4_window7_224_backend_TORCH | 0.78026 | 0.80696 | 0.0267 | ### Reason for changes New api in openvino to convert PT models ### Related tickets 117885
openvinotoolkit · Sep 19, 2023 · 50feea4 · 50feea4
1 parent 9c671f0
commit 50feea4
Show file tree

Hide file tree

Showing 10 changed files with 93 additions and 94 deletions.
diff --git a/tests/openvino/native/test_bias_correction.py b/tests/openvino/native/test_bias_correction.py
@@ -15,6 +15,7 @@
 import openvino.runtime as ov
 import pytest
 import torch
+from openvino.tools.mo import convert_model
 
 from nncf.common.factory import NNCFGraphFactory
 from nncf.openvino.graph.model_utils import remove_fq_from_inputs
@@ -24,7 +25,6 @@
 from tests.openvino.conftest import OPENVINO_NATIVE_TEST_ROOT
 from tests.openvino.native.common import compare_nncf_graphs
 from tests.post_training.test_templates.test_bias_correction import TemplateTestBCAlgorithm
-from tests.shared.command import Command
 
 
 class TestOVBCAlgorithm(TemplateTestBCAlgorithm):
@@ -40,11 +40,7 @@ def get_backend() -> OVBiasCorrectionAlgoBackend:
     def backend_specific_model(model: torch.nn.Module, tmp_dir: str):
         onnx_path = f"{tmp_dir}/model.onnx"
         torch.onnx.export(model, torch.rand(model.INPUT_SIZE), onnx_path, opset_version=13, input_names=["input.1"])
-        ov_path = f"{tmp_dir}/model.xml"
-        runner = Command(f"mo -m {onnx_path} -o {tmp_dir} -n model --compress_to_fp16=False")
-        runner.run()
-        core = ov.Core()
-        ov_model = core.read_model(ov_path)
+        ov_model = convert_model(onnx_path, input_shape=model.INPUT_SIZE, compress_to_fp16=False)
         return ov_model
 
     @staticmethod

diff --git a/tests/openvino/native/test_fast_bias_correction.py b/tests/openvino/native/test_fast_bias_correction.py
@@ -14,13 +14,13 @@
 import numpy as np
 import openvino.runtime as ov
 import torch
+from openvino.tools.mo import convert_model
 
 from nncf.common.factory import NNCFGraphFactory
 from nncf.openvino.graph.node_utils import get_bias_value
 from nncf.openvino.graph.node_utils import is_node_with_bias
 from nncf.quantization.algorithms.fast_bias_correction.openvino_backend import OVFastBiasCorrectionAlgoBackend
 from tests.post_training.test_templates.test_fast_bias_correction import TemplateTestFBCAlgorithm
-from tests.shared.command import Command
 
 
 class TestOVFBCAlgorithm(TemplateTestFBCAlgorithm):
@@ -36,11 +36,7 @@ def get_backend() -> OVFastBiasCorrectionAlgoBackend:
     def backend_specific_model(model: bool, tmp_dir: str):
         onnx_path = f"{tmp_dir}/model.onnx"
         torch.onnx.export(model, torch.rand(model.INPUT_SIZE), onnx_path, opset_version=13, input_names=["input.1"])
-        ov_path = f"{tmp_dir}/model.xml"
-        runner = Command(f"mo -m {onnx_path} -o {tmp_dir} -n model --compress_to_fp16=False")
-        runner.run()
-        core = ov.Core()
-        ov_model = core.read_model(ov_path)
+        ov_model = convert_model(onnx_path, input_shape=model.INPUT_SIZE, compress_to_fp16=False)
         return ov_model
 
     @staticmethod

diff --git a/tests/openvino/native/test_smooth_quant.py b/tests/openvino/native/test_smooth_quant.py
@@ -16,13 +16,13 @@
 import openvino.runtime as ov
 import pytest
 import torch
+from openvino.tools.mo import convert_model
 
 from nncf.openvino.graph.layer_attributes import OVLayerAttributes
 from nncf.openvino.graph.metatypes.openvino_metatypes import OVConvolutionMetatype
 from nncf.openvino.graph.metatypes.openvino_metatypes import OVMatMulMetatype
 from nncf.quantization.algorithms.smooth_quant.openvino_backend import OVSmoothQuantAlgoBackend
 from tests.post_training.test_templates.test_smooth_quant import TemplateTestSQAlgorithm
-from tests.shared.command import Command
 
 
 class TestOVSQAlgorithm(TemplateTestSQAlgorithm):
@@ -44,13 +44,10 @@ def get_backend() -> OVSmoothQuantAlgoBackend:
 
     @staticmethod
     def backend_specific_model(model: torch.nn.Module, tmp_dir: str) -> ov.Model:
+        # TODO(AlexanderDokuchaev): remove onnx export after fix 119625
         onnx_path = Path(f"{tmp_dir}/model.onnx")
         torch.onnx.export(model, torch.rand(model.INPUT_SIZE), onnx_path, opset_version=13, input_names=["input.1"])
-        ov_path = Path(f"{tmp_dir}/model.xml")
-        runner = Command(f"mo -m {onnx_path} -o {tmp_dir} -n model --compress_to_fp16=False")
-        runner.run()
-        core = ov.Core()
-        ov_model = core.read_model(ov_path)
+        ov_model = convert_model(onnx_path, input_shape=model.INPUT_SIZE, compress_to_fp16=False)
         return ov_model
 
     @staticmethod

diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py
@@ -14,7 +14,8 @@
 from nncf import ModelType
 from nncf import QuantizationPreset
 from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
-from tests.post_training.pipelines.base import ALL_NNCF_PTQ_BACKENDS
+from tests.post_training.pipelines.base import ALL_PTQ_BACKENDS
+from tests.post_training.pipelines.base import NNCF_PTQ_BACKENDS
 from tests.post_training.pipelines.base import BackendType
 from tests.post_training.pipelines.causal_language_model import CausalLMHF
 from tests.post_training.pipelines.image_classification_timm import ImageClassificationTimm
@@ -31,7 +32,7 @@
             "model_type": ModelType.TRANSFORMER,
             "subset_size": 2,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS + [BackendType.OPTIMUM],
+        "backends": ALL_PTQ_BACKENDS + [BackendType.OPTIMUM],
     },
     {
         "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
@@ -63,7 +64,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/deit3_small_patch16_224",
@@ -74,7 +75,7 @@
             "model_type": ModelType.TRANSFORMER,
             "advanced_parameters": AdvancedQuantizationParameters(smooth_quant_alpha=-1.0),
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/dla34",
@@ -83,7 +84,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/dpn68",
@@ -92,7 +93,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/efficientnet_b0",
@@ -101,7 +102,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/efficientnet_b0_BC",
@@ -120,7 +121,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/hrnet_w18",
@@ -129,14 +130,14 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/inception_resnet_v2",
         "model_id": "inception_resnet_v2",
         "pipeline_cls": ImageClassificationTimm,
         "ptq_params": {},
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": NNCF_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/levit_128",
@@ -147,7 +148,7 @@
             "model_type": ModelType.TRANSFORMER,
             "advanced_parameters": AdvancedQuantizationParameters(smooth_quant_alpha=0.05),
         },
-        "backends": [BackendType.TORCH, BackendType.ONNX, BackendType.OV, BackendType.POT],
+        "backends": [BackendType.TORCH, BackendType.ONNX, BackendType.OV],
     },
     {
         "reported_name": "timm/mobilenetv2_050",
@@ -156,7 +157,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/mobilenetv2_050_BC",
@@ -175,7 +176,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/regnetx_002",
@@ -184,7 +185,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/resnest14d",
@@ -193,14 +194,14 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/resnet18",
         "model_id": "resnet18",
         "pipeline_cls": ImageClassificationTimm,
         "ptq_params": {},
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/swin_base_patch4_window7_224",
@@ -210,7 +211,7 @@
             "preset": QuantizationPreset.MIXED,
             "model_type": ModelType.TRANSFORMER,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/tf_inception_v3",
@@ -219,14 +220,14 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/vgg11",
         "model_id": "vgg11",
         "pipeline_cls": ImageClassificationTimm,
         "ptq_params": {},
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": NNCF_PTQ_BACKENDS,
     },
     {
         "reported_name": "timm/visformer_small",
@@ -245,7 +246,7 @@
         "ptq_params": {
             "preset": QuantizationPreset.MIXED,
         },
-        "backends": ALL_NNCF_PTQ_BACKENDS,
+        "backends": ALL_PTQ_BACKENDS,
     },
 ]
 

diff --git a/tests/post_training/pipelines/base.py b/tests/post_training/pipelines/base.py
@@ -23,14 +23,13 @@
 import openvino.runtime as ov
 import torch
 from memory_profiler import memory_usage
+from openvino.tools.mo import convert_model
 from optimum.intel import OVQuantizer
-from torch import nn
 
 import nncf
 from nncf import TargetDevice
 from nncf.experimental.torch.quantization.quantize_model import quantize_impl as pt_impl_experimental
 from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
-from tests.shared.command import Command
 
 DEFAULT_VAL_THREADS = 4
 
@@ -45,7 +44,8 @@ class BackendType(Enum):
     OPTIMUM = "OPTIMUM"
 
 
-ALL_NNCF_PTQ_BACKENDS = [BackendType.OLD_TORCH, BackendType.TORCH, BackendType.ONNX, BackendType.OV, BackendType.POT]
+NNCF_PTQ_BACKENDS = [BackendType.OLD_TORCH, BackendType.TORCH, BackendType.ONNX, BackendType.OV]
+ALL_PTQ_BACKENDS = NNCF_PTQ_BACKENDS + [BackendType.POT]
 PT_BACKENDS = [BackendType.TORCH, BackendType.OLD_TORCH]
 OV_BACKENDS = [BackendType.OV, BackendType.POT, BackendType.OPTIMUM]
 
@@ -94,25 +94,6 @@ def get_result_dict(self):
         }
 
 
-def export_to_onnx(model: nn.Module, save_path: str, data_sample: torch.Tensor) -> None:
-    """
-    Export Torch model to ONNX format.
-    """
-    torch.onnx.export(model, data_sample, save_path, export_params=True, opset_version=13, do_constant_folding=False)
-
-
-def export_to_ir(model_path: str, save_path: str, model_name: str) -> None:
-    """
-    Export ONNX model to OpenVINO format.
-
-    :param model_path: Path to ONNX model.
-    :param save_path: Path directory to save OpenVINO IR model.
-    :param model_name: Model name.
-    """
-    runner = Command(f"mo -m {model_path} -o {save_path} -n {model_name} --compress_to_fp16=False")
-    runner.run()
-
-
 class BaseTestPipeline(ABC):
     """
     Base class to test post training quantization.
@@ -231,15 +212,15 @@ def save_quantized_model(self) -> None:
         if self.backend == BackendType.OPTIMUM:
             self.path_quantized_ir = self.output_model_dir / "openvino_model.xml"
         elif self.backend in PT_BACKENDS:
-            onnx_path = self.output_model_dir / "model.onnx"
-            export_to_onnx(self.quantized_model, str(onnx_path), self.dummy_tensor)
-            export_to_ir(onnx_path, self.output_model_dir, model_name="model")
+            ov_model = convert_model(self.quantized_model, example_input=self.dummy_tensor)
             self.path_quantized_ir = self.output_model_dir / "model.xml"
+            ov.serialize(ov_model, self.path_quantized_ir)
         elif self.backend == BackendType.ONNX:
             onnx_path = self.output_model_dir / "model.onnx"
             onnx.save(self.quantized_model, str(onnx_path))
-            export_to_ir(onnx_path, str(self.output_model_dir), model_name="model")
+            ov_model = convert_model(onnx_path)
             self.path_quantized_ir = self.output_model_dir / "model.xml"
+            ov.serialize(ov_model, self.path_quantized_ir)
         elif self.backend in OV_BACKENDS:
             self.path_quantized_ir = self.output_model_dir / "model.xml"
             ov.serialize(self.quantized_model, str(self.path_quantized_ir))

diff --git a/tests/post_training/pipelines/causal_language_model.py b/tests/post_training/pipelines/causal_language_model.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 
+import openvino.runtime as ov
 import transformers
 from optimum.intel import OVQuantizer
 from optimum.intel.openvino import OVModelForCausalLM
@@ -26,6 +27,7 @@ def prepare_model(self) -> None:
         if self.backend in OV_BACKENDS:
             self.model_hf = OVModelForCausalLM.from_pretrained(self.model_id, export=True, compile=False)
             self.model = self.model_hf.model
+            ov.serialize(self.model, self.output_model_dir / "model_fp32.xml")
 
     def prepare_preprocessor(self) -> None:
         self.preprocessor = transformers.AutoTokenizer.from_pretrained(self.model_id)