Unpin the version of tokenizers fix: #1849

hankcs · Oct 14, 2023 · 83412a1 · 83412a1
1 parent 698c7e8
commit 83412a1
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/setup.py b/setup.py
@@ -61,7 +61,6 @@
         'pynvml',
         'toposort==1.5',
         'transformers>=4.1.1',
-        'tokenizers==0.11.6',  # The latest tokenizers==0.12.1 failed to compile on macOS Python3.6
         'sentencepiece>=0.1.91',  # Essential for tokenization_bert_japanese
         'torch>=1.6.0',
         'hanlp-common>=0.0.19',