added tutorial for NeMo tokenizers (#1137)

* add tutorial Signed-off-by: Yang Zhang <[email protected]> * fix test Signed-off-by: Yang Zhang <[email protected]> * delete file Signed-off-by: Yang Zhang <[email protected]> * fix Signed-off-by: Yang Zhang <[email protected]> * split cells Signed-off-by: Yang Zhang <[email protected]>
NVIDIA · Sep 10, 2020 · 120550c · 120550c
1 parent f123997
commit 120550c
Show file tree

Hide file tree

Showing 3 changed files with 534 additions and 2 deletions.
diff --git a/README.rst b/README.rst
@@ -172,6 +172,9 @@ To run tutorials:
    * - ASR
      - Online Noise Augmentation
      - `05_Online_Noise_Augmentation.ipynb <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/asr/05_Online_Noise_Augmentation.ipynb>`_
+   * - NLP
+     - Exploring NeMo NLP Tokenizers
+     - `02_NLP_Tokenizers.ipynb <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/02_NLP_Tokenizers.ipynb>`_
    * - NLP
      - Token Classification (Named Entity Recognition)
      - `Token_Classification_Named_Entity_Recognition.ipynb <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb>`_

diff --git a/nemo/collections/nlp/modules/common/tokenizer_utils.py b/nemo/collections/nlp/modules/common/tokenizer_utils.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import nemo
 from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer
@@ -38,7 +38,7 @@ def get_tokenizer(
     tokenizer_name: str,
     tokenizer_model: Optional[str] = None,
     vocab_file: Optional[str] = None,
-    special_tokens: Optional[List[str]] = None,
+    special_tokens: Optional[Dict[str, str]] = None,
 ):
     """
     Args: