Ch 15.  BERT 미세조정학습.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57fe9bb8",
   "metadata": {},
   "source": [
    "# Ch 15. BERT 사전학습모형에 대한 미세조정학습\n",
    "\n",
    "## 15.1 BERT 학습을 위한 전처리\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "57ac8805",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a beautiful day! 토큰화 결과: ['what', 'a', 'beautiful', 'day', '!']\n",
      "Nvidia Titan XP has 12GB of VRAM 토큰화 결과: ['n', '##vid', '##ia', 'titan', 'xp', 'has', '12', '##gb', 'of', 'vr', '##am']\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "sentence1 = \"What a beautiful day!\"\n",
    "sentence2 = \"Nvidia Titan XP has 12GB of VRAM\"\n",
    "\n",
    "# 1. 토큰화 결과\n",
    "print(sentence1, '토큰화 결과:', tokenizer.tokenize(sentence1))\n",
    "print(sentence2, '토큰화 결과:', tokenizer.tokenize(sentence2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d017cf00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERT 입력: {'input_ids': [[101, 2054, 1037, 3376, 2154, 999, 102, 0, 0, 0, 0, 0, 0], [101, 1050, 17258, 2401, 16537, 26726, 2038, 2260, 18259, 1997, 27830, 3286, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
     ]
    }
   ],
   "source": [
    "# 2. BERT 모형 입력 생성\n",
    "inputs = tokenizer([sentence1, sentence2], padding=True)\n",
    "print('BERT 입력:', inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5ab20b20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "두 문장 시퀀스에 대한 BERT 입력: {'input_ids': [101, 2054, 1037, 3376, 2154, 999, 102, 1050, 17258, 2401, 16537, 26726, 2038, 2260, 18259, 1997, 27830, 3286, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n"
     ]
    }
   ],
   "source": [
    "# 3. 두 문장으로 이루어진 시퀀스에 대한 BERT 모형 입력 생성\n",
    "inputs = tokenizer(sentence1, sentence2, padding=True)\n",
    "print('두 문장 시퀀스에 대한 BERT 입력:', inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6ceb1e3",
   "metadata": {},
   "source": [
    "## 15.2 트랜스포머의 트레이너를 이용한 미세조정학습\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5bfdce73",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package movie_reviews to\n",
      "[nltk_data]     C:\\Users\\ParkSangun\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package movie_reviews is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set count:  1600\n",
      "Test set count:  400\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.corpus import movie_reviews\n",
    "from sklearn.model_selection import train_test_split #sklearn에서 제공하는 split 함수를 사용\n",
    "import numpy as np\n",
    "\n",
    "nltk.download('movie_reviews')\n",
    "fileids = movie_reviews.fileids() #movie review data에서 file id를 가져옴\n",
    "reviews = [movie_reviews.raw(fileid) for fileid in fileids] #file id를 이용해 raw text file을 가져옴\n",
    "categories = [movie_reviews.categories(fileid)[0] for fileid in fileids] \n",
    "\n",
    "# label을 0, 1의 값으로 변환\n",
    "label_dict = {'pos':1, 'neg':0}\n",
    "y = [label_dict[c] for c in categories]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(reviews, y, test_size=0.2, random_state=7)\n",
    "\n",
    "print('Train set count: ', len(X_train))\n",
    "print('Test set count: ', len(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4bbc145",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizerFast, BertForSequenceClassification \n",
    "\n",
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n",
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "train_input = tokenizer(X_train, truncation=True, padding=True, return_tensors=\"pt\")\n",
    "test_input = tokenizer(X_test, truncation=True, padding=True, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fc5ae8c",
   "metadata": {},
   "source": [
    "(1) https://tutorials.pytorch.kr/beginner/basics/data_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ecca3585",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class OurDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, inputs, labels):\n",
    "        self.inputs = inputs\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.inputs.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = OurDataset(train_input, y_train)\n",
    "test_dataset = OurDataset(test_input, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6c1f7ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install datasets\n",
    "\n",
    "#https://huggingface.co/docs/datasets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "07672dad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "metric = load_metric(\"accuracy\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c977367",
   "metadata": {},
   "source": [
    "(1) https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainingarguments#transformers.TrainingArguments\n",
    "\n",
    "(2) https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainingarguments#transformers.Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25fe706e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 1600\n",
      "  Num Epochs = 2\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 400\n",
      "C:\\Users\\PARKSA~1\\AppData\\Local\\Temp/ipykernel_2780/296560984.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.inputs.items()}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='400' max='400' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [400/400 02:47, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=400, training_loss=0.4372709655761719, metrics={'train_runtime': 170.2591, 'train_samples_per_second': 18.795, 'train_steps_per_second': 2.349, 'total_flos': 841955377152000.0, 'train_loss': 0.4372709655761719, 'epoch': 2.0})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Trainer에서 사용할 하이퍼 파라미터 지정\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',          # 모형 예측이나 체크포인트 출력 폴더, 반드시 필요함\n",
    "    num_train_epochs=2,              # 학습 에포크 수\n",
    "    per_device_train_batch_size=8,   # 학습에 사용할 배치 사이즈\n",
    "    per_device_eval_batch_size=16,   # 평가에 사용할 배치 사이즈\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,                     # 학습할 모형\n",
    "    args=training_args,              # 위에서 정의한 학습 매개변수\n",
    "    train_dataset=train_dataset,     # 훈련 데이터셋\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "# 미세조정학습 실행\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "86eee880",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 400\n",
      "  Batch size = 16\n",
      "C:\\Users\\PARKSA~1\\AppData\\Local\\Temp/ipykernel_2780/296560984.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.inputs.items()}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='25' max='25' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [25/25 00:06]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.3673644959926605,\n",
       " 'eval_accuracy': 0.87,\n",
       " 'eval_runtime': 6.8058,\n",
       " 'eval_samples_per_second': 58.774,\n",
       " 'eval_steps_per_second': 3.673,\n",
       " 'epoch': 2.0}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate(eval_dataset=test_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ba45e48",
   "metadata": {},
   "source": [
    "## 15.3 파이토치를 이용한 미세조정학습\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b89942fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "del model\n",
    "del trainer\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ad8f5c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_loader = DataLoader(train_dataset, shuffle=True, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "66ad3d85",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at C:\\Users\\ParkSangun/.cache\\huggingface\\transformers\\3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n",
      "Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.12.3\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin from cache at C:\\Users\\ParkSangun/.cache\\huggingface\\transformers\\a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of BertModel were initialized from the model checkpoint at bert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertModel\n",
    "\n",
    "bert_model = BertModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6768bcdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# BERT를 포함한 신경망 모형\n",
    "class MyModel(torch.nn.Module):\n",
    "    def __init__(self, pretrained_model, token_size, num_labels): \n",
    "        super(MyModel, self).__init__()\n",
    "        self.token_size = token_size\n",
    "        self.num_labels = num_labels\n",
    "        self.pretrained_model = pretrained_model\n",
    "\n",
    "        # 분류기 정의\n",
    "        self.classifier = torch.nn.Linear(self.token_size, self.num_labels)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        # BERT 모형에 입력을 넣고 출력을 받음\n",
    "        outputs = self.pretrained_model(**inputs)\n",
    "        # BERT 출력에서 CLS 토큰에 해당하는 부분만 가져옴\n",
    "        bert_clf_token = outputs.last_hidden_state[:,0,:]\n",
    "        \n",
    "        return self.classifier(bert_clf_token)\n",
    "\n",
    "# token_size는 BERT 토큰과 동일, bert_model.config.hidden_size로 알 수 있음\n",
    "model = MyModel(bert_model, num_labels=2, token_size=bert_model.config.hidden_size) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1d3f8220",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PARKSA~1\\AppData\\Local\\Temp/ipykernel_2780/296560984.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.inputs.items()}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, batch 100, elapsed time: 39.77, loss: 0.5169\n",
      "Epoch 1, batch 200, elapsed time: 80.67, loss: 0.3300\n",
      "Average loss for epoch 1: 0.5417\n",
      "Epoch 2, batch 100, elapsed time: 121.59, loss: 0.5303\n",
      "Epoch 2, batch 200, elapsed time: 162.57, loss: 0.1188\n",
      "Average loss for epoch 2: 0.2969\n",
      "Epoch 3, batch 100, elapsed time: 203.50, loss: 0.0132\n",
      "Epoch 3, batch 200, elapsed time: 244.48, loss: 0.0130\n",
      "Average loss for epoch 3: 0.1258\n",
      "Epoch 4, batch 100, elapsed time: 285.65, loss: 0.0095\n",
      "Epoch 4, batch 200, elapsed time: 327.03, loss: 0.0188\n",
      "Average loss for epoch 4: 0.1103\n"
     ]
    }
   ],
   "source": [
    "from transformers import AdamW\n",
    "import torch.nn.functional as F\n",
    "import time\n",
    "\n",
    "# GPU 가속을 사용할 수 있으면 device를 cuda로 설정하고, 아니면 cpu로 설정\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "model.to(device)  # 모형을 GPU로 복사\n",
    "model.train()     # 학습모드로 전환\n",
    "\n",
    "optim = AdamW(model.parameters(), lr=5e-5) # 옵티마이저를 트랜스포머가 제공하는 AdamW로 설정\n",
    "criterion = torch.nn.CrossEntropyLoss()    # 멀티클래스이므로 크로스 엔트로피를 손실함수로 사용\n",
    "\n",
    "start = time.time() # 시작시간 기록\n",
    "num_epochs = 4      # 학습 epoch를 4회로 설정\n",
    "for epoch in range(num_epochs):\n",
    "    total_epoch_loss = 0  # epoch의 총 loss 초기화\n",
    "    for step, batch in enumerate(train_loader):\n",
    "        optim.zero_grad()     # 그래디언트 초기화\n",
    "        # 배치에서 label을 제외한 입력만 추출하여  GPU로 복사\n",
    "        inputs = {k: v.to(device) for k, v in batch.items() if k != 'labels'} \n",
    "        labels = batch['labels'].to(device) # 배치에서 라벨을 추출하여 GPU로 복사\n",
    "        outputs = model(inputs) # 모형으로 결과 예측\n",
    "        # 두 클래스에 대해 예측하고 각각 비교해야 하므로 labels에 대해 원핫인코딩을 적용한 후에 손실을 게산\n",
    "        loss = criterion(outputs, F.one_hot(labels, num_classes=2).float()) # loss 계산\n",
    "\n",
    "        if (step+1) % 100 == 0: # 100 배치마다 경과한 시간과 loss를 출력\n",
    "            elapsed = time.time() - start\n",
    "            print('Epoch %d, batch %d, elapsed time: %.2f, loss: %.4f' % (epoch+1, step+1, elapsed, loss))\n",
    "        total_epoch_loss += loss\n",
    "        loss.backward() # 그래디언트 계산\n",
    "        optim.step()    # 가중치 업데이트\n",
    "    avg_epoch_loss = total_epoch_loss / len(train_loader) # epoch의 평균 loss 계산\n",
    "    print('Average loss for epoch %d: %.4f' % (epoch+1, avg_epoch_loss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e345ca4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PARKSA~1\\AppData\\Local\\Temp/ipykernel_2780/296560984.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.inputs.items()}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.8825}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "test_loader = DataLoader(test_dataset, batch_size=16)\n",
    "\n",
    "metric= load_metric(\"accuracy\")\n",
    "model.eval()\n",
    "for batch in test_loader:\n",
    "    inputs = {k: v.to(device) for k, v in batch.items() if k != 'labels'}\n",
    "    labels = batch['labels'].to(device)\n",
    "    \n",
    "    with torch.no_grad(): # 학습할 필요가 없으므로 그래디언트 계산을 끔\n",
    "        outputs = model(inputs)\n",
    "        #print(outputs)\n",
    "\n",
    "    predictions = torch.argmax(outputs, dim=-1)\n",
    "    metric.add_batch(predictions=predictions, references=labels)\n",
    "\n",
    "metric.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "847c525a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}