diff --git a/baseline1/run_train.py b/baseline1/run_train.py index 5961254..1e56453 100644 --- a/baseline1/run_train.py +++ b/baseline1/run_train.py @@ -22,7 +22,13 @@ ) from colorama import Fore, Style -# load_tokenizer(MAKE_TOKENIZER) +# To make tokenizer in new environment, please operate the code below (3 line) +''' +from medal_contender.tokenizer import load_tokenizer +from medal_contender.configs import MAKE_TOKENIZER +load_tokenizer(MAKE_TOKENIZER) +''' + from transformers.models.deberta_v2 import DebertaV2TokenizerFast red_font = Fore.RED @@ -36,6 +42,9 @@ # CUDA가 구체적인 에러를 보고하도록 설정 os.environ['CUDA_LAUNCH_BLOCKING'] = "1" +# Tokenizer parallelism을 사용하도록 환경 설정 +os.environ["TOKENIZERS_PARALLELISM"] = "true" + def run_training( model,