Add Subfolder Argument to CLI (#40)

openvinotoolkit · Feb 27, 2024 · 6b9fa42 · 6b9fa42
1 parent 930a3d3
commit 6b9fa42
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/python/openvino_tokenizers/cli.py b/python/openvino_tokenizers/cli.py
@@ -48,6 +48,16 @@ def get_parser() -> ArgumentParser:
         action="store_true",
         help="Add a detokenizer model to the output",
     )
+    parser.add_argument(
+        "--subfolder",
+        required=False,
+        type=str,
+        default="",
+        help=(
+            "Specify in case the tokenizer files are located inside a subfolder of the model repo on huggingface.co. "
+            "Example: `convert_tokenizer SimianLuo/LCM_Dreamshaper_v7 --subfolder tokenizer`"
+        ),
+    )
     parser.add_argument(
         "--skip-special-tokens",
         "--skip_special_tokens",
@@ -139,7 +149,7 @@ def convert_hf_tokenizer() -> None:
     args = get_parser().parse_args()
 
     print("Loading Huggingface Tokenizer...")
-    hf_tokenizer = AutoTokenizer.from_pretrained(args.name, trust_remote_code=args.trust_remote_code)
+    hf_tokenizer = AutoTokenizer.from_pretrained(args.name, subfolder=args.subfolder, trust_remote_code=args.trust_remote_code)
 
     print("Converting Huggingface Tokenizer to OpenVINO...")
     converted = convert_tokenizer(