[OpenVINO] Set Left Padding For Text Generation Task (#839)

* Set Left Padding For Text Gen Task

* Make Style

* Add OV Tokenizers Version Check

* Update tests/openvino/test_exporters_cli.py

Co-authored-by: Ilyas Moutawwakil <[email protected]>

* Make Style

* Make Style

---------

Co-authored-by: Ilyas Moutawwakil <[email protected]>

optimum/commands/export/openvino.py

@@ -305,7 +305,7 @@ def run(self):
             model = model_cls.from_pretrained(self.args.model, export=True, quantization_config=quantization_config)
             model.save_pretrained(self.args.output)
             if not self.args.disable_convert_tokenizer:
-                maybe_convert_tokenizers(library_name, self.args.output, model)
+                maybe_convert_tokenizers(library_name, self.args.output, model, task=task)
         elif task.startswith("text-generation") and quantize_with_dataset:
             from optimum.intel import OVModelForCausalLM
 
@@ -324,7 +324,7 @@ def run(self):
                 preprocessors = maybe_load_preprocessors(
                     self.args.model, trust_remote_code=self.args.trust_remote_code
                 )
-                maybe_convert_tokenizers(library_name, self.args.output, preprocessors=preprocessors)
+                maybe_convert_tokenizers(library_name, self.args.output, preprocessors=preprocessors, task=task)
         else:
             # TODO : add input shapes
             main_export(

optimum/exporters/openvino/__main__.py

@@ -387,7 +387,7 @@ class StoreAttr(object):
     )
 
     if convert_tokenizer:
-        maybe_convert_tokenizers(library_name, output, model, preprocessors)
+        maybe_convert_tokenizers(library_name, output, model, preprocessors, task=task)
 
     clear_class_registry()
     del model
@@ -399,7 +399,7 @@ class StoreAttr(object):
         GPTQQuantizer.post_init_model = orig_post_init_model
 
 
-def maybe_convert_tokenizers(library_name: str, output: Path, model=None, preprocessors=None):
+def maybe_convert_tokenizers(library_name: str, output: Path, model=None, preprocessors=None, task=None):
     """
     Tries to convert tokenizers to OV format and export them to disk.
 
@@ -412,6 +412,8 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             Model instance.
         preprocessors (`Iterable`, *optional*, defaults to None):
             Iterable possibly containing tokenizers to be converted.
+        task (`str`, *optional*, defaults to None):
+            The task to export the model for. Affects tokenizer conversion parameters.
     """
     from optimum.exporters.openvino.convert import export_tokenizer
 
@@ -420,7 +422,7 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             tokenizer = next(filter(lambda it: isinstance(it, PreTrainedTokenizerBase), preprocessors), None)
             if tokenizer:
                 try:
-                    export_tokenizer(tokenizer, output)
+                    export_tokenizer(tokenizer, output, task=task)
                 except Exception as exception:
                     logger.warning(
                         "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "
@@ -430,6 +432,6 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             for tokenizer_name in ("tokenizer", "tokenizer_2"):
                 tokenizer = getattr(model, tokenizer_name, None)
                 if tokenizer:
-                    export_tokenizer(tokenizer, output / tokenizer_name)
+                    export_tokenizer(tokenizer, output / tokenizer_name, task=task)
     else:
         logger.warning("Tokenizer won't be converted.")

optimum/exporters/openvino/convert.py

@@ -40,6 +40,7 @@
     _timm_version,
     _torch_version,
     _transformers_version,
+    compare_versions,
 )
 from optimum.utils import DEFAULT_DUMMY_SHAPES, is_diffusers_available
 from optimum.utils.save_utils import maybe_save_preprocessors
@@ -706,6 +707,7 @@ def export_tokenizer(
     tokenizer,
     output: Union[str, Path],
     suffix: Optional[str] = "",
+    task: Optional[str] = None,
 ):
     # avoid circular imports
     from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME
@@ -722,6 +724,15 @@ def export_tokenizer(
     if output.exists():
         tokenizer = maybe_convert_tokenizer_to_fast(tokenizer, output)
 
+    if (
+        task is not None
+        and task.startswith("text-generation")
+        and compare_versions("openvino-tokenizers", ">=", "2024.3.0.0")
+    ):
+        logger.info(f"Set tokenizer padding side to left for `{task}` task.")
+        tokenizer.padding_side = "left"
+        tokenizer.truncation_side = "left"
+
     try:
         converted = convert_tokenizer(tokenizer, with_detokenizer=True)
     except NotImplementedError:

optimum/intel/pipelines/pipeline_base.py

@@ -29,7 +29,6 @@
     ImageToTextPipeline,
     Pipeline,
     PreTrainedTokenizer,
-    PreTrainedTokenizerFast,
     QuestionAnsweringPipeline,
     SummarizationPipeline,
     Text2TextGenerationPipeline,

tests/openvino/test_exporters_cli.py

@@ -41,7 +41,10 @@
 )
 from optimum.intel.openvino.configuration import _DEFAULT_4BIT_CONFIGS
 from optimum.intel.openvino.utils import _HEAD_TO_AUTOMODELS
-from optimum.intel.utils.import_utils import is_openvino_tokenizers_available
+from optimum.intel.utils.import_utils import (
+    compare_versions,
+    is_openvino_tokenizers_available,
+)
 
 
 class OVCLIExportTestCase(unittest.TestCase):
@@ -171,6 +174,9 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
 
+            if task.startswith("text-generation") and compare_versions("openvino-tokenizers", ">=", "2024.3.0.0"):
+                self.assertIn("Set tokenizer padding side to left", output)
+
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
     def test_exporters_cli_fp16(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir: