From cc7b3d287df4255a77d9fd255e301672ab8da60f Mon Sep 17 00:00:00 2001 From: Lyalyushkin Nikolay Date: Wed, 28 Aug 2024 21:06:46 +0200 Subject: [PATCH] Added statefull GPTNeox for PTQ conformance tests (#2923) ### Changes Added statefull GPTNeox for PTQ conformance tests ### Reason for changes There was a WA for the bug ref-135375 that used stateless instead of stateful model type. https://github.com/openvinotoolkit/nncf/pull/2569/files#diff-99cb9f0934f8c3ee6a920ad98a7de38df18ebb07acd15b97a6e32dc318c17168R29 Both modes should be tested with new OpenVINO release. ### Related tickets ref-137079 ### Tests - [x] job/NNCF/job/manual/job/post_training_quantization/466/ --- tests/post_training/data/ptq_reference_data.yaml | 8 ++++++-- tests/post_training/model_scope.py | 15 ++++++++++++++- .../pipelines/causal_language_model.py | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/post_training/data/ptq_reference_data.yaml b/tests/post_training/data/ptq_reference_data.yaml index 4976a7134b1..4bacf77ff2e 100644 --- a/tests/post_training/data/ptq_reference_data.yaml +++ b/tests/post_training/data/ptq_reference_data.yaml @@ -10,9 +10,13 @@ hf/bert-base-uncased_backend_OV: metric_value: null hf/bert-base-uncased_backend_TORCH: metric_value: null -hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_backend_FP32: +hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_FP32: metric_value: null -hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_backend_OPTIMUM: +hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_FP32: + metric_value: null +hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_OPTIMUM: + metric_value: null +hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_OPTIMUM: metric_value: null hf/hf-internal-testing/tiny-random-gpt2_backend_FP32: metric_value: null diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py index be968547618..ccaaec8e7c6 100644 --- a/tests/post_training/model_scope.py +++ b/tests/post_training/model_scope.py @@ -46,7 +46,7 @@ "backends": ALL_PTQ_BACKENDS + [BackendType.OPTIMUM], }, { - "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull", "model_id": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", "pipeline_cls": CausalLMHF, "compression_params": { @@ -54,6 +54,19 @@ "model_type": ModelType.TRANSFORMER, "subset_size": 2, }, + "params": {"is_stateful": True}, + "backends": [BackendType.OPTIMUM], + }, + { + "reported_name": "hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless", + "model_id": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "pipeline_cls": CausalLMHF, + "compression_params": { + "preset": QuantizationPreset.MIXED, + "model_type": ModelType.TRANSFORMER, + "subset_size": 2, + }, + "params": {"is_stateful": False}, "backends": [BackendType.OPTIMUM], }, { diff --git a/tests/post_training/pipelines/causal_language_model.py b/tests/post_training/pipelines/causal_language_model.py index e2df5b5a47e..17761196c46 100644 --- a/tests/post_training/pipelines/causal_language_model.py +++ b/tests/post_training/pipelines/causal_language_model.py @@ -24,9 +24,10 @@ class CausalLMHF(PTQTestPipeline): """Pipeline for causal language models from Hugging Face repository""" def prepare_model(self) -> None: + is_stateful = self.params.get("is_stateful", False) if self.backend in OV_BACKENDS + [BackendType.FP32]: self.model_hf = OVModelForCausalLM.from_pretrained( - self.model_id, export=True, compile=False, stateful=False + self.model_id, export=True, compile=False, stateful=is_stateful ) self.model = self.model_hf.model ov.serialize(self.model, self.fp32_model_dir / "model_fp32.xml")