Added statefull GPTNeox for PTQ conformance tests (#2923)

### Changes Added statefull GPTNeox for PTQ conformance tests ### Reason for changes There was a WA for the bug ref-135375 that used stateless instead of stateful model type. https://github.com/openvinotoolkit/nncf/pull/2569/files#diff-99cb9f0934f8c3ee6a920ad98a7de38df18ebb07acd15b97a6e32dc318c17168R29 Both modes should be tested with new OpenVINO release. ### Related tickets ref-137079 ### Tests - [x] job/NNCF/job/manual/job/post_training_quantization/466/
openvinotoolkit · Aug 28, 2024 · cc7b3d2 · cc7b3d2
1 parent cf06bc1
commit cc7b3d2
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 4 deletions.
diff --git a/tests/post_training/data/ptq_reference_data.yaml b/tests/post_training/data/ptq_reference_data.yaml
@@ -10,9 +10,13 @@ hf/bert-base-uncased_backend_OV:
   metric_value: null
 hf/bert-base-uncased_backend_TORCH:
   metric_value: null
-hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_backend_FP32:
+hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_FP32:
   metric_value: null
-hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_backend_OPTIMUM:
+hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_FP32:
+  metric_value: null
+hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_OPTIMUM:
+  metric_value: null
+hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_OPTIMUM:
   metric_value: null
 hf/hf-internal-testing/tiny-random-gpt2_backend_FP32:
   metric_value: null

diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py
@@ -46,14 +46,27 @@
         "backends": ALL_PTQ_BACKENDS + [BackendType.OPTIMUM],
     },
     {
-        "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
+        "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull",
         "model_id": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
         "pipeline_cls": CausalLMHF,
         "compression_params": {
             "preset": QuantizationPreset.MIXED,
             "model_type": ModelType.TRANSFORMER,
             "subset_size": 2,
         },
+        "params": {"is_stateful": True},
+        "backends": [BackendType.OPTIMUM],
+    },
+    {
+        "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless",
+        "model_id": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
+        "pipeline_cls": CausalLMHF,
+        "compression_params": {
+            "preset": QuantizationPreset.MIXED,
+            "model_type": ModelType.TRANSFORMER,
+            "subset_size": 2,
+        },
+        "params": {"is_stateful": False},
         "backends": [BackendType.OPTIMUM],
     },
     {

diff --git a/tests/post_training/pipelines/causal_language_model.py b/tests/post_training/pipelines/causal_language_model.py
@@ -24,9 +24,10 @@ class CausalLMHF(PTQTestPipeline):
     """Pipeline for causal language models from Hugging Face repository"""
 
     def prepare_model(self) -> None:
+        is_stateful = self.params.get("is_stateful", False)
         if self.backend in OV_BACKENDS + [BackendType.FP32]:
             self.model_hf = OVModelForCausalLM.from_pretrained(
-                self.model_id, export=True, compile=False, stateful=False
+                self.model_id, export=True, compile=False, stateful=is_stateful
             )
             self.model = self.model_hf.model
             ov.serialize(self.model, self.fp32_model_dir / "model_fp32.xml")