Add chat_template for tokenizer extracted from GGUF model (#32908)

* add chat_template to gguf tokenizer * add template through tokenizer config
huggingface · Aug 22, 2024 · ee8c01f · ee8c01f
1 parent 99d67f1
commit ee8c01f
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py
@@ -121,8 +121,10 @@ def __init__(self, *args, **kwargs):
             gguf_param = load_gguf_checkpoint(kwargs.get("vocab_file"))
             architecture = gguf_param["config"]["model_type"]
             tokenizer_dict = gguf_param["tokenizer"]
+            tokenizer_config = gguf_param["tokenizer_config"]
             fast_tokenizer, additional_kwargs = convert_gguf_tokenizer(architecture, tokenizer_dict)
 
+            kwargs.update(tokenizer_config)
             if len(additional_kwargs) > 0:
                 kwargs.update(additional_kwargs)