From 0f4077e541a29cfbf7e1e7bfea4d109b94fc322d Mon Sep 17 00:00:00 2001
From: HyeonhoonLee <jackli0373@gmail.com>
Date: Fri, 11 Mar 2022 16:07:31 +0900
Subject: [PATCH] feat: edit making tokenizers

---
 baseline1/run_train.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/baseline1/run_train.py b/baseline1/run_train.py
index 5961254..1e56453 100644
--- a/baseline1/run_train.py
+++ b/baseline1/run_train.py
@@ -22,7 +22,13 @@
 )
 from colorama import Fore, Style
 
-# load_tokenizer(MAKE_TOKENIZER)
+# To make tokenizer in new environment, please operate the code below (3 line)
+'''
+from medal_contender.tokenizer import load_tokenizer
+from medal_contender.configs import MAKE_TOKENIZER
+load_tokenizer(MAKE_TOKENIZER)
+'''
+
 from transformers.models.deberta_v2 import DebertaV2TokenizerFast
 
 red_font = Fore.RED
@@ -36,6 +42,9 @@
 # CUDA가 구체적인 에러를 보고하도록 설정
 os.environ['CUDA_LAUNCH_BLOCKING'] = "1"
 
+# Tokenizer parallelism을 사용하도록 환경 설정
+os.environ["TOKENIZERS_PARALLELISM"] = "true" 
+
 
 def run_training(
     model,