hai2023-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3ac7ea1e",
   "metadata": {
    "id": "rxoBTI7SCcWm",
    "papermill": {
     "duration": 0.007187,
     "end_time": "2023-08-20T12:42:54.435996",
     "exception": false,
     "start_time": "2023-08-20T12:42:54.428809",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 입력된 발화의 지역 방언을 구분하는 모델 학습시키기\n",
    "\n",
    "- 입력된 텍스트가 표준어 발화인지 아니면 특정 지역의 방언인지 분류할 수 있는 모델을 학습시켜 봅시다.\n",
    "- 먼저 필요한 라이브러리를 설치 및 import해 줍니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4c66881",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:42:54.452833Z",
     "iopub.status.busy": "2023-08-20T12:42:54.452388Z",
     "iopub.status.idle": "2023-08-20T12:43:18.610240Z",
     "shell.execute_reply": "2023-08-20T12:43:18.609256Z"
    },
    "id": "yzHvV4HlGw4r",
    "outputId": "b70e57dc-c019-45ea-a800-e93a2cb2cb7f",
    "papermill": {
     "duration": 24.169443,
     "end_time": "2023-08-20T12:43:18.612847",
     "exception": false,
     "start_time": "2023-08-20T12:42:54.443404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n",
      "  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n",
      "  warnings.warn(f\"file system plugins are not loaded: {e}\")\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers easydict keras_preprocessing --quiet\n",
    "\n",
    "import os\n",
    "import random\n",
    "import easydict\n",
    "import requests\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "from transformers import ElectraTokenizer, ElectraForSequenceClassification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8a4c9af",
   "metadata": {
    "id": "sBSOt-66CcWq",
    "papermill": {
     "duration": 0.006809,
     "end_time": "2023-08-20T12:43:18.626953",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.620144",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 다음은 학습 과정에서 데이터의 전처리와 배치 단위 입력을 수월하게 처리해줄 수 있게 하는 DataLoader를 이용하여 모델 학습을 위한 데이터를 전처리하는 함수입니다.\n",
    "- generate_data_loader를 호출하면 입력된 파일 경로에서 파일을 읽어와 적절한 토크나이징을 진행하고 args에 정의되어 있는 크기만큼 배치 단위로 데이터를 제공할 수 있는 iteratable한 DataLoader 객체를 반환하게 됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2da759b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.642822Z",
     "iopub.status.busy": "2023-08-20T12:43:18.641814Z",
     "iopub.status.idle": "2023-08-20T12:43:18.652600Z",
     "shell.execute_reply": "2023-08-20T12:43:18.651642Z"
    },
    "id": "G8mG8ElvGw4t",
    "papermill": {
     "duration": 0.021043,
     "end_time": "2023-08-20T12:43:18.654735",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.633692",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_data_loader(file_path, tokenizer, args):\n",
    "    def get_input_ids(data):\n",
    "        document_bert = [\"[CLS] \" + str(s) + \" [SEP]\" for s in data]\n",
    "        tokenized_texts = [tokenizer.tokenize(s) for s in tqdm(document_bert, \"Tokenizing\")]\n",
    "        input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tqdm(tokenized_texts, \"Converting tokens to ids\")]\n",
    "        print(\"Padding sequences...\")\n",
    "        input_ids = pad_sequences(input_ids, maxlen=args.maxlen, dtype='long', truncating='post', padding='post')\n",
    "        return input_ids\n",
    "\n",
    "    def get_attention_masks(input_ids):\n",
    "        attention_masks = []\n",
    "        for seq in tqdm(input_ids, \"Generating attention masks\"):\n",
    "            seq_mask = [float(i > 0) for i in seq]\n",
    "            attention_masks.append(seq_mask)\n",
    "        return attention_masks\n",
    "\n",
    "    def get_data_loader(inputs, masks, labels, batch_size=args.batch):\n",
    "        data = TensorDataset(torch.tensor(inputs), torch.tensor(masks), torch.tensor(labels))\n",
    "        sampler = RandomSampler(data) if args.mode == 'train' else SequentialSampler(data)\n",
    "        data_loader = DataLoader(data, sampler=sampler, batch_size=batch_size)\n",
    "        return data_loader\n",
    "\n",
    "    data_df = pd.read_csv(file_path)\n",
    "    input_ids = get_input_ids(data_df['text'].values)\n",
    "    attention_masks = get_attention_masks(input_ids)\n",
    "    data_loader = get_data_loader(input_ids, attention_masks, data_df['label'].values if args.mode=='train' else [-1]*len(data_df))\n",
    "\n",
    "    return data_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca500a09",
   "metadata": {
    "id": "98fSFgXPCcWs",
    "papermill": {
     "duration": 0.006729,
     "end_time": "2023-08-20T12:43:18.668361",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.661632",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 아래 함수는 모델을 학습/추론하는 과정에서 필요한 보조 함수들입니다.\n",
    "- save는 torch 라이브러리의 state_dict를 저장하는 기능을 이용해 모델의 가중치만 주어진 경로에 저장하는 함수입니다.\n",
    "- flat_accuracy는 모델이 예측한 결과값과 정답 라벨을 비교하여 얼마나 정확하게 맞혔는지 정확도를 구해주는 함수입니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e0276f82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.683131Z",
     "iopub.status.busy": "2023-08-20T12:43:18.682831Z",
     "iopub.status.idle": "2023-08-20T12:43:18.688403Z",
     "shell.execute_reply": "2023-08-20T12:43:18.687514Z"
    },
    "id": "TfO9ISW2Gw4u",
    "papermill": {
     "duration": 0.01538,
     "end_time": "2023-08-20T12:43:18.690437",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.675057",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def save(model, dir_name):\n",
    "    os.makedirs(dir_name, exist_ok=True)\n",
    "    torch.save(model.state_dict(), os.path.join(dir_name, 'model.pth'))\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9178f7b2",
   "metadata": {
    "id": "19xHdV2yCcWt",
    "papermill": {
     "duration": 0.006647,
     "end_time": "2023-08-20T12:43:18.704150",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.697503",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- predict는 학습된 모델을 평가하기 위한 함수입니다. 데이터 입력을 DataLoader 형식으로 받아 모델이 예측한 값을 받아온 뒤 flat_accuracy를 호출하여 정답 라벨과 비교한 정확도를 계산합니다.\n",
    "- 모델의 추론 과정(Validation 또는 Test 과정)에서 back propagation은 일어나지 않기 때문에, 계산 속도를 높이기 위해 torch.no_grad()를 실행하여 모델에 데이터를 입력해도 gradient가 따로 계산되어 저장되지 않도록 했습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ac69dde",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.719303Z",
     "iopub.status.busy": "2023-08-20T12:43:18.719021Z",
     "iopub.status.idle": "2023-08-20T12:43:18.726466Z",
     "shell.execute_reply": "2023-08-20T12:43:18.725519Z"
    },
    "id": "wqQfejZyGw4u",
    "papermill": {
     "duration": 0.017558,
     "end_time": "2023-08-20T12:43:18.728756",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.711198",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def predict(model, args, data_loader):\n",
    "    print('start predict')\n",
    "    model.eval()\n",
    "\n",
    "    eval_accuracy = []\n",
    "    logits = []\n",
    "\n",
    "    for step, batch in tqdm(enumerate(data_loader)):\n",
    "        batch = tuple(t.to(args.device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        with torch.no_grad():\n",
    "            outputs = model(b_input_ids,\n",
    "                            attention_mask=b_input_mask)\n",
    "        logit = outputs[0]\n",
    "\n",
    "        logit = logit.detach().cpu().numpy()\n",
    "        label = b_labels.cpu().numpy()\n",
    "\n",
    "        logits.append(logit)\n",
    "\n",
    "        accuracy = flat_accuracy(logit, label)\n",
    "        eval_accuracy.append(accuracy)\n",
    "\n",
    "    logits = np.vstack(logits)\n",
    "    predict_labels = np.argmax(logits, axis=1)\n",
    "    return predict_labels, np.mean(eval_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0469164e",
   "metadata": {
    "id": "eRlgMVV-CcWu",
    "papermill": {
     "duration": 0.006707,
     "end_time": "2023-08-20T12:43:18.743113",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.736406",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 이 노트북에서 가장 중요한 부분인 train은 모델을 학습시키기 위한 함수입니다. Train data와 Valid data를 각각 DataLoader 형태로 입력받아 학습과 검증 과정을 거치게 됩니다.\n",
    "- 개선된 optimization 알고리즘인 AdamW와 learning rate를 선형적으로 감소시키는 linear scheduler를 이용하여 학습을 진행합니다.\n",
    "- 한 epoch가 종료되면 valid_loader를 이용해 predict를 호출하여 validation accuracy를 계산합니다.\n",
    "- 대부분의 PyTorch를 활용한 모델 학습 과정은 이 함수와 비슷한 과정을 거쳐 진행되니 패턴에 익숙해지면 좋습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b23c40c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.757783Z",
     "iopub.status.busy": "2023-08-20T12:43:18.757496Z",
     "iopub.status.idle": "2023-08-20T12:43:18.768445Z",
     "shell.execute_reply": "2023-08-20T12:43:18.767541Z"
    },
    "id": "twbi-hjmGw4v",
    "papermill": {
     "duration": 0.020767,
     "end_time": "2023-08-20T12:43:18.770607",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.749840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train(model, args, train_loader, valid_loader):\n",
    "    optimizer = AdamW(model.parameters(),\n",
    "                      lr=args.lr,\n",
    "                      eps=args.eps\n",
    "                      )\n",
    "    total_steps = len(train_loader) * args.epochs\n",
    "\n",
    "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
    "                                                num_warmup_steps=0,\n",
    "                                                num_training_steps=total_steps)\n",
    "\n",
    "    seed_val = 42\n",
    "    random.seed(seed_val)\n",
    "    np.random.seed(seed_val)\n",
    "    torch.manual_seed(seed_val)\n",
    "    torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "    print('start training')\n",
    "    for epoch in range(args.epochs):\n",
    "        model.train()\n",
    "        train_loss = []\n",
    "        for step, batch in tqdm(enumerate(train_loader), f\"training epoch {epoch}\", total=len(train_loader)):\n",
    "            model.zero_grad()\n",
    "            batch = tuple(t.to(args.device) for t in batch)\n",
    "            b_input_ids, b_input_mask, b_labels = batch\n",
    "            outputs = model(b_input_ids,\n",
    "                            attention_mask=b_input_mask,\n",
    "                            labels=b_labels)\n",
    "            loss = outputs[0]\n",
    "            train_loss.append(loss.item())\n",
    "            loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "            optimizer.step()\n",
    "            scheduler.step()\n",
    "\n",
    "        avg_train_loss = np.mean(train_loss)\n",
    "        _, avg_train_accuracy = predict(model, args, train_loader)\n",
    "        _, avg_val_accuracy = predict(model, args, valid_loader)\n",
    "        print(\"Epoch {0},  Average training loss: {1:.4f} , Train accuracy : {2:.4f}, Validation accuracy : {3:.4f}\"\\\n",
    "              .format(epoch, avg_train_loss, avg_train_accuracy, avg_val_accuracy))\n",
    "\n",
    "        save(model, \"./saved_checkpoints/\" + str(epoch))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24d3583c",
   "metadata": {
    "id": "eQ94RwlbCcWv",
    "papermill": {
     "duration": 0.006655,
     "end_time": "2023-08-20T12:43:18.784054",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.777399",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 필요한 함수를 정의 완료했으니 학습을 본격적으로 진행해 봅시다.\n",
    "- args에는 학습 과정에서 지정해야 할 각종 하이퍼파라미터(배치 사이즈, learning rate 등등)와 데이터 파일 경로 등을 입력해둬 코드 실행 과정에서 사용할 수 있도록 합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5421c38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.799464Z",
     "iopub.status.busy": "2023-08-20T12:43:18.799168Z",
     "iopub.status.idle": "2023-08-20T12:43:18.859621Z",
     "shell.execute_reply": "2023-08-20T12:43:18.858479Z"
    },
    "id": "soqhgEExGw4w",
    "papermill": {
     "duration": 0.070639,
     "end_time": "2023-08-20T12:43:18.861684",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.791045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "args = easydict.EasyDict({\n",
    "  \"train_path\" : \"./data/train.csv\",\n",
    "  \"valid_path\" : \"./data/valid.csv\",\n",
    "  \"device\" : 'cpu',\n",
    "  \"mode\" : \"train\",\n",
    "  \"batch\" : 128,\n",
    "  \"maxlen\" : 128,\n",
    "  \"lr\" : 5e-5,\n",
    "  \"eps\" : 1e-8,\n",
    "  \"epochs\" : 5,\n",
    "  \"model_ckpt\" : \"monologg/koelectra-small-v3-discriminator\",\n",
    "})\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    args.device = 'cuda'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f397bd84",
   "metadata": {
    "id": "iQakEB52CcWv",
    "papermill": {
     "duration": 0.006557,
     "end_time": "2023-08-20T12:43:18.875289",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.868732",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 전처리가 완료된 데이터는 kaggle competition의 Data 페이지에서 다운로드받을 수 있습니다.\n",
    "- 다운로드받은 데이터 파일을 압축 해제해서 `data` 디렉토리에 옮겨 주세요.\n",
    "- kaggle notebook을 사용중인 경우 이미 데이터가 다운로드되어 있으니 바로 사용하면 됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "18588f6d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:18.891254Z",
     "iopub.status.busy": "2023-08-20T12:43:18.889795Z",
     "iopub.status.idle": "2023-08-20T12:43:20.251519Z",
     "shell.execute_reply": "2023-08-20T12:43:20.250223Z"
    },
    "id": "TOPktFqTJu-m",
    "outputId": "10c1823c-553e-468f-f7ac-c27679514c2e",
    "papermill": {
     "duration": 1.372389,
     "end_time": "2023-08-20T12:43:20.254341",
     "exception": false,
     "start_time": "2023-08-20T12:43:18.881952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 현재 디렉토리 내부 data 폴더에 데이터 파일을 다운로드받아 압축을 해제해 주세요.\n",
    "os.makedirs('./data', exist_ok=True)\n",
    "\n",
    "# kaggle competition에서 제공하는 notebook을 사용 중인 경우\n",
    "!cp /kaggle/input/hai2023summer/* ./data\n",
    "\n",
    "# 데이터 다운로드 링크\n",
    "# https://www.kaggle.com/competitions/hai2023summer/data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f13745c",
   "metadata": {
    "id": "hlCscJoxCcWw",
    "papermill": {
     "duration": 0.006661,
     "end_time": "2023-08-20T12:43:20.268432",
     "exception": false,
     "start_time": "2023-08-20T12:43:20.261771",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- Train/Valid 데이터는 각각 idx, text, label 세 개의 column을 가지는 csv 파일 형태로 이루어져 있습니다. text는 평가 대상 발화 텍스트가, label은 방언 여부를 나타내는 정수 라벨입니다(표준어일 경우 0, 경상도 방언일 경우 1, 제주도 방언일 경우 2)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cb7dcfae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:20.283790Z",
     "iopub.status.busy": "2023-08-20T12:43:20.283433Z",
     "iopub.status.idle": "2023-08-20T12:43:20.675215Z",
     "shell.execute_reply": "2023-08-20T12:43:20.673364Z"
    },
    "id": "Fj1jfARMCcWw",
    "outputId": "07eabadd-cfb2-45ba-b945-79f016df9172",
    "papermill": {
     "duration": 0.402476,
     "end_time": "2023-08-20T12:43:20.677807",
     "exception": false,
     "start_time": "2023-08-20T12:43:20.275331",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   idx                                               text  label\n",
      "0    0                                   어 뭐라하지 엄청 정들이 많아      0\n",
      "1    1                  대개 보니까 엄마 나이가 돼서 아프시니까 허리 수술을 이렇게      0\n",
      "2    2                     에피소드라면 일단은 버스가 네가 알다시피 되게 크잖아.      0\n",
      "3    3  무신 죽었져 어쩌게 기저 질환이 이신 사람들이 그렇게 했겠지만은 맹심향 좀 몸이 이...      2\n",
      "4    4  그 쌤이 처음에 다른 샘들한테 내 욕을 하는 걸 내가 건너서 들었는데 나중에는 제일...      0\n",
      "           idx                                               text  label\n",
      "190735  190735                                     아이들양 간식을 메길거난.      2\n",
      "190736  190736                                               그니까요      0\n",
      "190737  190737  아~ 뭐~ 올래도 갑자기 올래는 이상하게 십일월 달인가 되게 안 추웠잖아요 십일월 ...      1\n",
      "190738  190738                      다 나왔잖아 치얼업이랑 티티랑 그때 노래가 너무 좋아      0\n",
      "190739  190739         고양이상인 사람은 예쁜데 고양이를 보면서 막 이러진 않아 막 너는 막 이렇게      0\n"
     ]
    }
   ],
   "source": [
    "train_data_df = pd.read_csv(args.train_path)\n",
    "print(train_data_df.head())\n",
    "print(train_data_df.tail())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c43641ad",
   "metadata": {
    "id": "ScKz85rzCcWw",
    "papermill": {
     "duration": 0.00668,
     "end_time": "2023-08-20T12:43:20.693146",
     "exception": false,
     "start_time": "2023-08-20T12:43:20.686466",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 이제 학습을 위한 모델을 준비해 보겠습니다.\n",
    "- args에 정의되어 있는 모델의 체크포인트를 이용해 Huggingface hub로부터 sequence classification을 위한 모델과 토크나이저를 불러온 뒤, 모델을 GPU 메모리로 옮깁니다.\n",
    "- 분류해야 하는 class의 종류가 3가지(표준어, 경상도, 제주도)이기 때문에, num_labels를 3으로 지정해 주었습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ffb5de20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:20.708110Z",
     "iopub.status.busy": "2023-08-20T12:43:20.707796Z",
     "iopub.status.idle": "2023-08-20T12:43:35.540428Z",
     "shell.execute_reply": "2023-08-20T12:43:35.539468Z"
    },
    "id": "iy2KULr5J6mE",
    "outputId": "56ee358f-e138-4d9f-cede-5ed8686b3d8b",
    "papermill": {
     "duration": 14.842731,
     "end_time": "2023-08-20T12:43:35.542671",
     "exception": false,
     "start_time": "2023-08-20T12:43:20.699940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23a30cae5bfd4f8096af72efb0eaaf53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/458 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4ec75b5dc1b4056b5d9985b940529dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/56.6M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at monologg/koelectra-small-v3-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense.weight']\n",
      "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at monologg/koelectra-small-v3-discriminator and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b87008fe68ef4ec9b520f2529f9eb848",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/263k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47f8733db1a141c39ed993aa9ae196c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/61.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# load model and tokenizer\n",
    "model = ElectraForSequenceClassification.from_pretrained(args.model_ckpt, num_labels=3)\n",
    "model.to(args.device)\n",
    "tokenizer = ElectraTokenizer.from_pretrained(args.model_ckpt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d7a7d59",
   "metadata": {
    "id": "VGFn0menCcWx",
    "papermill": {
     "duration": 0.007662,
     "end_time": "2023-08-20T12:43:35.558474",
     "exception": false,
     "start_time": "2023-08-20T12:43:35.550812",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 미리 정의된 generate_data_loader 함수를 이용해 train/valid 데이터에 대한 DataLoader를 생성합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cbe87498",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:43:35.577297Z",
     "iopub.status.busy": "2023-08-20T12:43:35.576993Z",
     "iopub.status.idle": "2023-08-20T12:44:49.836472Z",
     "shell.execute_reply": "2023-08-20T12:44:49.835096Z"
    },
    "id": "xMAhDeUlJ7-6",
    "outputId": "9212cc75-7047-4881-d4a7-59ace55b75bb",
    "papermill": {
     "duration": 74.273228,
     "end_time": "2023-08-20T12:44:49.839435",
     "exception": false,
     "start_time": "2023-08-20T12:43:35.566207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Tokenizing: 100%|██████████| 190740/190740 [00:43<00:00, 4430.23it/s]\n",
      "Converting tokens to ids: 100%|██████████| 190740/190740 [00:03<00:00, 49008.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding sequences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating attention masks: 100%|██████████| 190740/190740 [00:18<00:00, 10412.67it/s]\n",
      "Tokenizing: 100%|██████████| 973/973 [00:00<00:00, 4711.79it/s]\n",
      "Converting tokens to ids: 100%|██████████| 973/973 [00:00<00:00, 48006.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding sequences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating attention masks: 100%|██████████| 973/973 [00:00<00:00, 11624.33it/s]\n"
     ]
    }
   ],
   "source": [
    "train_dataloader = generate_data_loader(args.train_path, tokenizer, args)\n",
    "validation_dataloader = generate_data_loader(args.valid_path, tokenizer, args)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "614ccfbb",
   "metadata": {
    "id": "UEAt94NRCcWx",
    "papermill": {
     "duration": 0.05841,
     "end_time": "2023-08-20T12:44:49.955836",
     "exception": false,
     "start_time": "2023-08-20T12:44:49.897426",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 모델과 하이퍼파라미터 그리고 데이터가 준비되었으니 학습을 진행시켜 봅시다.\n",
    "- 각 epoch가 끝날 때 마다 모델의 가중치를 저장하고 validation 결과를 출력합니다. 이를 바탕으로 최적의 결과를 가지는 모델을 선택할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e11625a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T12:44:50.070504Z",
     "iopub.status.busy": "2023-08-20T12:44:50.070042Z",
     "iopub.status.idle": "2023-08-20T13:25:15.388620Z",
     "shell.execute_reply": "2023-08-20T13:25:15.387511Z"
    },
    "papermill": {
     "duration": 2425.37936,
     "end_time": "2023-08-20T13:25:15.391280",
     "exception": false,
     "start_time": "2023-08-20T12:44:50.011920",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training epoch 0: 100%|██████████| 1491/1491 [06:13<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1491it [01:52, 13.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:00, 13.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0,  Average training loss: 0.3537 , Train accuracy : 0.9310, Validation accuracy : 0.9180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training epoch 1: 100%|██████████| 1491/1491 [06:11<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1491it [01:52, 13.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:00, 13.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1,  Average training loss: 0.2092 , Train accuracy : 0.9457, Validation accuracy : 0.9226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training epoch 2: 100%|██████████| 1491/1491 [06:11<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1491it [01:52, 13.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:00, 13.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2,  Average training loss: 0.1782 , Train accuracy : 0.9536, Validation accuracy : 0.9232\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training epoch 3: 100%|██████████| 1491/1491 [06:11<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1491it [01:52, 13.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:00, 13.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3,  Average training loss: 0.1575 , Train accuracy : 0.9588, Validation accuracy : 0.9281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training epoch 4: 100%|██████████| 1491/1491 [06:11<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1491it [01:52, 13.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:00, 13.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4,  Average training loss: 0.1438 , Train accuracy : 0.9611, Validation accuracy : 0.9258\n"
     ]
    }
   ],
   "source": [
    "model = train(model, args, train_dataloader, validation_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f3c916f",
   "metadata": {
    "papermill": {
     "duration": 0.967262,
     "end_time": "2023-08-20T13:25:17.270269",
     "exception": false,
     "start_time": "2023-08-20T13:25:16.303007",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 학습이 완료된 모델을 활용하여 test 데이터셋에 대한 추론을 진행해 보겠습니다.\n",
    "- 평가를 위해서는 데이터의 각 인덱스에 맞춰 라벨을 예측하는 것이 필요하기 때문에, 새로운 argument를 정의해 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "01c77795",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T13:25:19.710570Z",
     "iopub.status.busy": "2023-08-20T13:25:19.710203Z",
     "iopub.status.idle": "2023-08-20T13:25:20.366138Z",
     "shell.execute_reply": "2023-08-20T13:25:20.364254Z"
    },
    "papermill": {
     "duration": 2.116084,
     "end_time": "2023-08-20T13:25:20.368820",
     "exception": false,
     "start_time": "2023-08-20T13:25:18.252736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Tokenizing: 100%|██████████| 626/626 [00:00<00:00, 4172.40it/s]\n",
      "Converting tokens to ids: 100%|██████████| 626/626 [00:00<00:00, 44138.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding sequences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating attention masks: 100%|██████████| 626/626 [00:00<00:00, 11695.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:00, 13.11it/s]\n"
     ]
    }
   ],
   "source": [
    "test_args = easydict.EasyDict({\n",
    "  \"device\" : \"cpu\",\n",
    "  \"mode\" : \"test\",\n",
    "  \"batch\" : 128,\n",
    "  \"maxlen\" : 128,\n",
    "})\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    test_args.device = 'cuda'\n",
    "\n",
    "test_dataloader = generate_data_loader(\"data/test.csv\", tokenizer=tokenizer, args=test_args)\n",
    "labels, _ = predict(model, test_args, test_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e36345d",
   "metadata": {
    "papermill": {
     "duration": 0.912735,
     "end_time": "2023-08-20T13:25:22.193484",
     "exception": false,
     "start_time": "2023-08-20T13:25:21.280749",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- 추론 결과는 데이터의 인덱스와 합쳐 제출 파일 형식에 맞게 저장합니다.\n",
    "- kaggle competition 페이지에서 submit prediction 메뉴를 활용해 예측 결과에 대한 평가를 확인할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0d6b6157",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T13:25:24.114331Z",
     "iopub.status.busy": "2023-08-20T13:25:24.113979Z",
     "iopub.status.idle": "2023-08-20T13:25:24.125188Z",
     "shell.execute_reply": "2023-08-20T13:25:24.124205Z"
    },
    "papermill": {
     "duration": 0.909785,
     "end_time": "2023-08-20T13:25:24.127112",
     "exception": false,
     "start_time": "2023-08-20T13:25:23.217327",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submit_df = pd.DataFrame()\n",
    "submit_df[\"idx\"] = range(len(labels))\n",
    "submit_df[\"label\"] = labels\n",
    "submit_df.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b0d3eb05",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-20T13:25:25.996215Z",
     "iopub.status.busy": "2023-08-20T13:25:25.995484Z",
     "iopub.status.idle": "2023-08-20T13:25:26.009523Z",
     "shell.execute_reply": "2023-08-20T13:25:26.008643Z"
    },
    "papermill": {
     "duration": 0.925532,
     "end_time": "2023-08-20T13:25:26.011612",
     "exception": false,
     "start_time": "2023-08-20T13:25:25.086080",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>621</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>622</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623</th>\n",
       "      <td>623</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>624</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>626 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     idx  label\n",
       "0      0      0\n",
       "1      1      2\n",
       "2      2      2\n",
       "3      3      0\n",
       "4      4      0\n",
       "..   ...    ...\n",
       "621  621      0\n",
       "622  622      0\n",
       "623  623      0\n",
       "624  624      0\n",
       "625  625      1\n",
       "\n",
       "[626 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 2566.363017,
   "end_time": "2023-08-20T13:25:30.421818",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-08-20T12:42:44.058801",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "00bcebbfa4f84bec8245a90116f891ed": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "05a6cb4e183d4a31951b08782a2fae58": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0ccecbd315db4a9eb19d948575cd9ce2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0f60f6e2c578402e8426012bbde36fce": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b9f3ae1a44324c43ace3561c4f096f63",
       "placeholder": "​",
       "style": "IPY_MODEL_1a8b825922f244be91551075c5507fb8",
       "value": " 56.6M/56.6M [00:04&lt;00:00, 16.5MB/s]"
      }
     },
     "11eeac950940430d8d16f1460adbe35d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "16c0b7c328ee4f62ad578535cc032128": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1a8b825922f244be91551075c5507fb8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1e822ff748e7486592ab198d095f96d2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1f7f7db25cf44da984a8ee2f1a4ec4b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_54e0178d936546f5882ac41e86ad796f",
       "placeholder": "​",
       "style": "IPY_MODEL_81fc3744d6a1442187ad015a8dc36610",
       "value": "Downloading (…)okenizer_config.json: 100%"
      }
     },
     "20b5384266bd4fa9b12bdf91484e8b4f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "23a30cae5bfd4f8096af72efb0eaaf53": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_2d75ca12cf08426d8101216b4c06237a",
        "IPY_MODEL_a2486c5c19654bcd91df754c626bf9b5",
        "IPY_MODEL_436e489fff0a492ea4886482e67ec8cf"
       ],
       "layout": "IPY_MODEL_0ccecbd315db4a9eb19d948575cd9ce2"
      }
     },
     "2c26aff307ef4c7dac6cc6c379548e88": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_20b5384266bd4fa9b12bdf91484e8b4f",
       "placeholder": "​",
       "style": "IPY_MODEL_b9e7adfe756245799d2e9d5b195932c8",
       "value": " 61.0/61.0 [00:00&lt;00:00, 4.12kB/s]"
      }
     },
     "2d75ca12cf08426d8101216b4c06237a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_da705beeb71047c68c8150c02abfe21c",
       "placeholder": "​",
       "style": "IPY_MODEL_11eeac950940430d8d16f1460adbe35d",
       "value": "Downloading (…)lve/main/config.json: 100%"
      }
     },
     "2e1d65b4fe214c41bdecda4651d28a47": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "35884f573c984a2390860ae262f3d911": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "398242e82893460ead5e5b3c0fbbfdc9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "436e489fff0a492ea4886482e67ec8cf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_00bcebbfa4f84bec8245a90116f891ed",
       "placeholder": "​",
       "style": "IPY_MODEL_16c0b7c328ee4f62ad578535cc032128",
       "value": " 458/458 [00:00&lt;00:00, 28.8kB/s]"
      }
     },
     "47f8733db1a141c39ed993aa9ae196c9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_1f7f7db25cf44da984a8ee2f1a4ec4b0",
        "IPY_MODEL_6c162224d04c413996961ccf3c295681",
        "IPY_MODEL_2c26aff307ef4c7dac6cc6c379548e88"
       ],
       "layout": "IPY_MODEL_97fc31d2f2b24483842fa35d4d0723b8"
      }
     },
     "54e0178d936546f5882ac41e86ad796f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5599a5df1a344946a28fe506f36b388f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5cbbf956815949549977607f6b88ee29": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5f04bffc4cd344818632c62661f98404": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "60cf894c3c9e48eaa4dcd398650ef83d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "66e93371889a4b54a7b0669f32f51638": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_05a6cb4e183d4a31951b08782a2fae58",
       "placeholder": "​",
       "style": "IPY_MODEL_8b8f3ae47a1442ef802d6ef42dbf1986",
       "value": "Downloading pytorch_model.bin: 100%"
      }
     },
     "6c162224d04c413996961ccf3c295681": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5599a5df1a344946a28fe506f36b388f",
       "max": 61.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e599551dc0754866ac1a1edf8611692c",
       "value": 61.0
      }
     },
     "726b01a70f224be190161387480ab3b8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "72be0e1a16094a65bc3676d15d633223": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_398242e82893460ead5e5b3c0fbbfdc9",
       "placeholder": "​",
       "style": "IPY_MODEL_726b01a70f224be190161387480ab3b8",
       "value": " 263k/263k [00:00&lt;00:00, 452kB/s]"
      }
     },
     "81fc3744d6a1442187ad015a8dc36610": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "8679bca2481149aeba7892e368487046": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8b8f3ae47a1442ef802d6ef42dbf1986": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "97fc31d2f2b24483842fa35d4d0723b8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9e3b369c23914d589d75ea8360dd1cfb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a2486c5c19654bcd91df754c626bf9b5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_60cf894c3c9e48eaa4dcd398650ef83d",
       "max": 458.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b21c5d280fb54338ac1a54a1a98264bb",
       "value": 458.0
      }
     },
     "a7b8d5f310f1448099ba62419039e897": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5cbbf956815949549977607f6b88ee29",
       "max": 56577499.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_35884f573c984a2390860ae262f3d911",
       "value": 56577499.0
      }
     },
     "b21c5d280fb54338ac1a54a1a98264bb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "b87008fe68ef4ec9b520f2529f9eb848": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d8b05916b0e5477f9c7febab45d3069d",
        "IPY_MODEL_bf2ed0efb9964457ada44fc09b84f304",
        "IPY_MODEL_72be0e1a16094a65bc3676d15d633223"
       ],
       "layout": "IPY_MODEL_1e822ff748e7486592ab198d095f96d2"
      }
     },
     "b9e7adfe756245799d2e9d5b195932c8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b9f3ae1a44324c43ace3561c4f096f63": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bf2ed0efb9964457ada44fc09b84f304": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9e3b369c23914d589d75ea8360dd1cfb",
       "max": 263326.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2e1d65b4fe214c41bdecda4651d28a47",
       "value": 263326.0
      }
     },
     "d2b6e8c9700341b88cbb6a1c719802ce": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d8b05916b0e5477f9c7febab45d3069d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8679bca2481149aeba7892e368487046",
       "placeholder": "​",
       "style": "IPY_MODEL_5f04bffc4cd344818632c62661f98404",
       "value": "Downloading (…)solve/main/vocab.txt: 100%"
      }
     },
     "da705beeb71047c68c8150c02abfe21c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e599551dc0754866ac1a1edf8611692c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "f4ec75b5dc1b4056b5d9985b940529dc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_66e93371889a4b54a7b0669f32f51638",
        "IPY_MODEL_a7b8d5f310f1448099ba62419039e897",
        "IPY_MODEL_0f60f6e2c578402e8426012bbde36fce"
       ],
       "layout": "IPY_MODEL_d2b6e8c9700341b88cbb6a1c719802ce"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}