Add ernie-tiny configs to ERNIE tokenizer (PaddlePaddle#406)

Add ernie-tiny configs to ERNIE tokenizer Co-authored-by: Guo Sheng <[email protected]>
wangxicoding · May 18, 2021 · 2e64244 · 2e64244
1 parent 55e96ba
commit 2e64244
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/paddlenlp/transformers/ernie/tokenizer.py b/paddlenlp/transformers/ernie/tokenizer.py
@@ -72,6 +72,8 @@ class ErnieTokenizer(PretrainedTokenizer):
         "vocab_file": {
             "ernie-1.0":
             "https://paddlenlp.bj.bcebos.com/models/transformers/ernie/vocab.txt",
+            "ernie-tiny":
+            "https://paddlenlp.bj.bcebos.com/models/transformers/ernie_tiny/vocab.txt",
             "ernie-2.0-en":
             "https://paddlenlp.bj.bcebos.com/models/transformers/ernie_v2_base/vocab.txt",
             "ernie-2.0-large-en":
@@ -88,6 +90,9 @@ class ErnieTokenizer(PretrainedTokenizer):
         "ernie-1.0": {
             "do_lower_case": True
         },
+        "ernie-tiny": {
+            "do_lower_case": True
+        },
         "ernie-2.0-en": {
             "do_lower_case": True
         },