Support for additional_special_tokens. cfg.special_tokens may be None.

axolotl-ai-cloud · Jan 30, 2024 · db66577 · db66577
1 parent 29dbde0
commit db66577
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
@@ -161,10 +161,11 @@ def load_tokenizer(cfg):
             if getattr(tokenizer, attr_name) is None:
                 setattr(tokenizer, attr_name, "<|endoftext|>")
 
-    additional_special_tokens = cfg.special_tokens.pop(
-        "additional_special_tokens", None
-    )
+    additional_special_tokens = None
     if cfg.special_tokens:
+        additional_special_tokens = cfg.special_tokens.pop(
+            "additional_special_tokens", None
+        )
         lora_modules_to_save = get_linear_embedding_layers(model_config.model_type)
         for k, val in cfg.special_tokens.items():
             # check if new special token is not already in tokenizer and