npr_MC2_electra.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Mini-Challenge 2 npr\n",
    "## Q&A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlnet-base-cased were not used when initializing XLNetForQuestionAnsweringSimple: ['lm_loss.weight', 'lm_loss.bias']\n",
      "- This IS expected if you are initializing XLNetForQuestionAnsweringSimple from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLNetForQuestionAnsweringSimple from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLNetForQuestionAnsweringSimple were not initialized from the model checkpoint at xlnet-base-cased and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datasets import load_dataset, load_metric\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering, TrainingArguments, Trainer, default_data_collator\n",
    "from utils import *\n",
    "\n",
    "config = load_yaml_file('electra_config.yaml')\n",
    "\n",
    "##\n",
    "GPU_USAGE = config['GPU_USAGE']\n",
    "BATCH_SIZE = config['BATCH_SIZE']\n",
    "MAX_LENGTH = config['MAX_LENGTH']\n",
    "DOC_STRIDE = config['DOC_STRIDE']\n",
    "##\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"google/electra-small-discriminator\")\n",
    "model = AutoModelForQuestionAnswering.from_pretrained(\"google/electra-small-discriminator\").to(\"cuda:0\")\n",
    "\n",
    "PAD_RIGHT = tokenizer.padding_side == \"right\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset squad_v2 (C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d)\n",
      "100%|██████████| 2/2 [00:00<00:00, 82.85it/s]\n"
     ]
    }
   ],
   "source": [
    "data = load_dataset(\"squad_v2\")\n",
    "# with open('data/train-v2.0.json') as f:\n",
    "#     data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-f7f821988ca865d4.arrow\n",
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-c9925728a42415f7.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = data.map(\n",
    "    prepare_train_features, \n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT,\n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        }, \n",
    "    batched=True,\n",
    "    remove_columns=data[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n",
      "***** Running training *****\n",
      "  Num examples = 137985\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 51747\n",
      "  1%|          | 500/51747 [02:55<5:02:51,  2.82it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.3331, 'learning_rate': 1.9806752082246313e-05, 'epoch': 0.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-500\\special_tokens_map.json\n",
      "  2%|▏         | 1000/51747 [05:56<4:55:55,  2.86it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-1000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-1000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.8253, 'learning_rate': 1.9613504164492628e-05, 'epoch': 0.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-1000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-1000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-1000\\special_tokens_map.json\n",
      "  3%|▎         | 1500/51747 [08:52<4:51:30,  2.87it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-1500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-1500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6944, 'learning_rate': 1.9420256246738943e-05, 'epoch': 0.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-1500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-1500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-1500\\special_tokens_map.json\n",
      "  4%|▍         | 2000/51747 [11:48<4:47:41,  2.88it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-2000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-2000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.655, 'learning_rate': 1.9227008328985258e-05, 'epoch': 0.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-2000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-2000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-2000\\special_tokens_map.json\n",
      "  5%|▍         | 2500/51747 [14:41<4:30:01,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-2500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-2500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6227, 'learning_rate': 1.903376041123157e-05, 'epoch': 0.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-2500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-2500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-2500\\special_tokens_map.json\n",
      "  6%|▌         | 3000/51747 [17:27<4:27:02,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-3000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-3000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5831, 'learning_rate': 1.8840512493477885e-05, 'epoch': 0.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-3000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-3000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-3000\\special_tokens_map.json\n",
      "  7%|▋         | 3500/51747 [20:13<4:23:35,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-3500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-3500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5548, 'learning_rate': 1.8647264575724196e-05, 'epoch': 0.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-3500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-3500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-3500\\special_tokens_map.json\n",
      "  8%|▊         | 4000/51747 [22:59<4:20:26,  3.06it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-4000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-4000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.553, 'learning_rate': 1.845401665797051e-05, 'epoch': 0.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-4000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-4000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-4000\\special_tokens_map.json\n",
      "  9%|▊         | 4500/51747 [25:47<4:20:50,  3.02it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-4500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-4500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5394, 'learning_rate': 1.8260768740216826e-05, 'epoch': 0.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-4500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-4500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-4500\\special_tokens_map.json\n",
      " 10%|▉         | 5000/51747 [28:36<4:17:07,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-5000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-5000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5163, 'learning_rate': 1.806752082246314e-05, 'epoch': 0.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-5000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-5000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-5000\\special_tokens_map.json\n",
      " 11%|█         | 5500/51747 [31:25<4:17:42,  2.99it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-5500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-5500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4965, 'learning_rate': 1.7874272904709453e-05, 'epoch': 0.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-5500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-5500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-5500\\special_tokens_map.json\n",
      " 12%|█▏        | 6000/51747 [34:15<4:12:29,  3.02it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-6000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-6000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5317, 'learning_rate': 1.7681024986955764e-05, 'epoch': 0.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-6000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-6000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-6000\\special_tokens_map.json\n",
      " 13%|█▎        | 6500/51747 [37:03<4:08:44,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-6500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-6500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4826, 'learning_rate': 1.748777706920208e-05, 'epoch': 0.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-6500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-6500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-6500\\special_tokens_map.json\n",
      " 14%|█▎        | 7000/51747 [39:50<4:06:18,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-7000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-7000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5095, 'learning_rate': 1.7294529151448394e-05, 'epoch': 0.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-7000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-7000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-7000\\special_tokens_map.json\n",
      " 14%|█▍        | 7500/51747 [42:38<4:03:08,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-7500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-7500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5089, 'learning_rate': 1.710128123369471e-05, 'epoch': 0.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-7500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-7500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-7500\\special_tokens_map.json\n",
      " 15%|█▌        | 8000/51747 [45:25<4:02:44,  3.00it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-8000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-8000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4959, 'learning_rate': 1.690803331594102e-05, 'epoch': 0.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-8000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-8000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-8000\\special_tokens_map.json\n",
      " 16%|█▋        | 8500/51747 [48:13<3:57:54,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-8500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-8500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4921, 'learning_rate': 1.6714785398187336e-05, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-8500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-8500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-8500\\special_tokens_map.json\n",
      " 17%|█▋        | 9000/51747 [51:00<3:53:55,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-9000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-9000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4889, 'learning_rate': 1.652153748043365e-05, 'epoch': 0.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-9000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-9000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-9000\\special_tokens_map.json\n",
      " 18%|█▊        | 9500/51747 [53:47<3:52:34,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-9500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-9500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.446, 'learning_rate': 1.6328289562679966e-05, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-9500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-9500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-9500\\special_tokens_map.json\n",
      " 19%|█▉        | 10000/51747 [56:34<3:48:58,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-10000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-10000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4793, 'learning_rate': 1.6135041644926278e-05, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-10000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-10000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-10000\\special_tokens_map.json\n",
      " 20%|██        | 10500/51747 [59:22<3:46:30,  3.03it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-10500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-10500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4411, 'learning_rate': 1.5941793727172592e-05, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-10500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-10500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-10500\\special_tokens_map.json\n",
      " 21%|██▏       | 11000/51747 [1:02:09<3:43:17,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-11000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-11000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4186, 'learning_rate': 1.5748545809418904e-05, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-11000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-11000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-11000\\special_tokens_map.json\n",
      " 22%|██▏       | 11500/51747 [1:04:56<3:40:57,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-11500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-11500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4309, 'learning_rate': 1.555529789166522e-05, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-11500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-11500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-11500\\special_tokens_map.json\n",
      " 23%|██▎       | 12000/51747 [1:07:43<3:37:43,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-12000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-12000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.473, 'learning_rate': 1.5362049973911534e-05, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-12000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-12000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-12000\\special_tokens_map.json\n",
      " 24%|██▍       | 12500/51747 [1:10:29<3:34:44,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-12500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-12500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4442, 'learning_rate': 1.5168802056157846e-05, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-12500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-12500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-12500\\special_tokens_map.json\n",
      " 25%|██▌       | 13000/51747 [1:13:16<3:32:07,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-13000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-13000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4615, 'learning_rate': 1.497555413840416e-05, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-13000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-13000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-13000\\special_tokens_map.json\n",
      " 26%|██▌       | 13500/51747 [1:16:03<3:29:10,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-13500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-13500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.447, 'learning_rate': 1.4782306220650474e-05, 'epoch': 0.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-13500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-13500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-13500\\special_tokens_map.json\n",
      " 27%|██▋       | 14000/51747 [1:18:49<3:26:25,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-14000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-14000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4438, 'learning_rate': 1.4589058302896789e-05, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-14000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-14000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-14000\\special_tokens_map.json\n",
      " 28%|██▊       | 14500/51747 [1:21:35<3:24:12,  3.04it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-14500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-14500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4415, 'learning_rate': 1.43958103851431e-05, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-14500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-14500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-14500\\special_tokens_map.json\n",
      " 29%|██▉       | 15000/51747 [1:24:22<3:20:42,  3.05it/s] Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-15000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-15000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4334, 'learning_rate': 1.4202562467389415e-05, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-15000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-15000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-15000\\special_tokens_map.json\n",
      " 30%|██▉       | 15500/51747 [1:27:08<3:18:01,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-15500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-15500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.422, 'learning_rate': 1.4009314549635729e-05, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-15500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-15500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-15500\\special_tokens_map.json\n",
      " 31%|███       | 16000/51747 [1:29:54<3:15:18,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-16000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-16000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4443, 'learning_rate': 1.3816066631882044e-05, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-16000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-16000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-16000\\special_tokens_map.json\n",
      " 32%|███▏      | 16500/51747 [1:32:40<3:12:28,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-16500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-16500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4176, 'learning_rate': 1.3622818714128357e-05, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-16500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-16500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-16500\\special_tokens_map.json\n",
      " 33%|███▎      | 17000/51747 [1:35:26<3:09:40,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-17000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-17000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.419, 'learning_rate': 1.3429570796374669e-05, 'epoch': 0.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-17000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-17000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-17000\\special_tokens_map.json\n",
      " 33%|███▎      | 17249/51747 [1:36:50<2:29:46,  3.84it/s] ***** Running Evaluation *****\n",
      "  Num examples = 12852\n",
      "  Batch size = 8\n",
      "\n",
      " 33%|███▎      | 17249/51747 [1:40:27<2:29:46,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.414121150970459, 'eval_runtime': 216.7308, 'eval_samples_per_second': 59.299, 'eval_steps_per_second': 7.415, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 17500/51747 [1:41:49<3:07:30,  3.04it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-17500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-17500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3755, 'learning_rate': 1.3236322878620984e-05, 'epoch': 1.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-17500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-17500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-17500\\special_tokens_map.json\n",
      " 35%|███▍      | 18000/51747 [1:44:36<3:04:27,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-18000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-18000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3441, 'learning_rate': 1.3043074960867297e-05, 'epoch': 1.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-18000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-18000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-18000\\special_tokens_map.json\n",
      " 36%|███▌      | 18500/51747 [1:47:22<3:02:08,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-18500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-18500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3294, 'learning_rate': 1.2849827043113612e-05, 'epoch': 1.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-18500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-18500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-18500\\special_tokens_map.json\n",
      " 37%|███▋      | 19000/51747 [1:50:08<2:59:08,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-19000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-19000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3187, 'learning_rate': 1.2656579125359925e-05, 'epoch': 1.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-19000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-19000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-19000\\special_tokens_map.json\n",
      " 38%|███▊      | 19500/51747 [1:52:55<2:56:20,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-19500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-19500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3262, 'learning_rate': 1.246333120760624e-05, 'epoch': 1.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-19500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-19500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-19500\\special_tokens_map.json\n",
      " 39%|███▊      | 20000/51747 [1:55:41<2:53:24,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-20000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-20000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3223, 'learning_rate': 1.2270083289852552e-05, 'epoch': 1.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-20000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-20000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-20000\\special_tokens_map.json\n",
      " 40%|███▉      | 20500/51747 [1:58:27<2:50:47,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-20500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-20500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3388, 'learning_rate': 1.2076835372098867e-05, 'epoch': 1.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-20500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-20500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-20500\\special_tokens_map.json\n",
      " 41%|████      | 21000/51747 [2:01:14<2:48:08,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-21000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-21000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3149, 'learning_rate': 1.188358745434518e-05, 'epoch': 1.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-21000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-21000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-21000\\special_tokens_map.json\n",
      " 42%|████▏     | 21500/51747 [2:04:01<2:45:26,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-21500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-21500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3362, 'learning_rate': 1.1690339536591495e-05, 'epoch': 1.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-21500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-21500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-21500\\special_tokens_map.json\n",
      " 43%|████▎     | 22000/51747 [2:06:48<2:42:57,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-22000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-22000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3474, 'learning_rate': 1.1497091618837808e-05, 'epoch': 1.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-22000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-22000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-22000\\special_tokens_map.json\n",
      " 43%|████▎     | 22500/51747 [2:09:34<2:40:34,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-22500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-22500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3389, 'learning_rate': 1.1303843701084121e-05, 'epoch': 1.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-22500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-22500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-22500\\special_tokens_map.json\n",
      " 44%|████▍     | 23000/51747 [2:12:21<2:36:54,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-23000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-23000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3336, 'learning_rate': 1.1110595783330436e-05, 'epoch': 1.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-23000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-23000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-23000\\special_tokens_map.json\n",
      " 45%|████▌     | 23500/51747 [2:15:07<2:35:08,  3.03it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-23500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-23500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.325, 'learning_rate': 1.0917347865576748e-05, 'epoch': 1.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-23500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-23500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-23500\\special_tokens_map.json\n",
      " 46%|████▋     | 24000/51747 [2:17:54<2:32:00,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-24000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-24000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3179, 'learning_rate': 1.0724099947823063e-05, 'epoch': 1.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-24000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-24000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-24000\\special_tokens_map.json\n",
      " 47%|████▋     | 24500/51747 [2:20:40<2:29:04,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-24500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-24500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3221, 'learning_rate': 1.0530852030069376e-05, 'epoch': 1.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-24500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-24500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-24500\\special_tokens_map.json\n",
      " 48%|████▊     | 25000/51747 [2:23:27<2:26:19,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-25000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-25000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3439, 'learning_rate': 1.0337604112315691e-05, 'epoch': 1.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-25000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-25000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-25000\\special_tokens_map.json\n",
      " 49%|████▉     | 25500/51747 [2:26:13<2:23:31,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-25500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-25500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3307, 'learning_rate': 1.0144356194562005e-05, 'epoch': 1.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-25500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-25500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-25500\\special_tokens_map.json\n",
      " 50%|█████     | 26000/51747 [2:29:00<2:20:51,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-26000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-26000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2958, 'learning_rate': 9.951108276808318e-06, 'epoch': 1.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-26000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-26000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-26000\\special_tokens_map.json\n",
      " 51%|█████     | 26500/51747 [2:31:47<2:18:03,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-26500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-26500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3341, 'learning_rate': 9.757860359054631e-06, 'epoch': 1.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-26500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-26500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-26500\\special_tokens_map.json\n",
      " 52%|█████▏    | 27000/51747 [2:34:33<2:15:18,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-27000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-27000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2976, 'learning_rate': 9.564612441300946e-06, 'epoch': 1.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-27000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-27000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-27000\\special_tokens_map.json\n",
      " 53%|█████▎    | 27500/51747 [2:37:19<2:12:14,  3.06it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-27500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-27500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3058, 'learning_rate': 9.37136452354726e-06, 'epoch': 1.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-27500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-27500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-27500\\special_tokens_map.json\n",
      " 54%|█████▍    | 28000/51747 [2:40:06<2:09:49,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-28000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-28000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3077, 'learning_rate': 9.178116605793573e-06, 'epoch': 1.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-28000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-28000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-28000\\special_tokens_map.json\n",
      " 55%|█████▌    | 28500/51747 [2:42:52<2:07:25,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-28500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-28500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3404, 'learning_rate': 8.984868688039888e-06, 'epoch': 1.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-28500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-28500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-28500\\special_tokens_map.json\n",
      " 56%|█████▌    | 29000/51747 [2:45:39<2:04:14,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-29000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-29000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3387, 'learning_rate': 8.791620770286201e-06, 'epoch': 1.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-29000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-29000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-29000\\special_tokens_map.json\n",
      " 57%|█████▋    | 29500/51747 [2:48:25<2:01:55,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-29500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-29500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3213, 'learning_rate': 8.598372852532514e-06, 'epoch': 1.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-29500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-29500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-29500\\special_tokens_map.json\n",
      " 58%|█████▊    | 30000/51747 [2:51:12<1:59:07,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-30000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-30000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3144, 'learning_rate': 8.40512493477883e-06, 'epoch': 1.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-30000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-30000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-30000\\special_tokens_map.json\n",
      " 59%|█████▉    | 30500/51747 [2:54:00<1:56:11,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-30500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-30500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3299, 'learning_rate': 8.211877017025143e-06, 'epoch': 1.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-30500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-30500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-30500\\special_tokens_map.json\n",
      " 60%|█████▉    | 31000/51747 [2:56:46<1:53:20,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-31000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-31000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3102, 'learning_rate': 8.018629099271456e-06, 'epoch': 1.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-31000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-31000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-31000\\special_tokens_map.json\n",
      " 61%|██████    | 31500/51747 [2:59:32<1:50:48,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-31500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-31500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3085, 'learning_rate': 7.825381181517769e-06, 'epoch': 1.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-31500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-31500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-31500\\special_tokens_map.json\n",
      " 62%|██████▏   | 32000/51747 [3:02:19<1:48:14,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-32000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-32000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3244, 'learning_rate': 7.632133263764084e-06, 'epoch': 1.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-32000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-32000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-32000\\special_tokens_map.json\n",
      " 63%|██████▎   | 32500/51747 [3:05:05<1:45:06,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-32500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-32500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3449, 'learning_rate': 7.438885346010397e-06, 'epoch': 1.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-32500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-32500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-32500\\special_tokens_map.json\n",
      " 64%|██████▍   | 33000/51747 [3:07:52<1:42:38,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-33000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-33000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3029, 'learning_rate': 7.245637428256711e-06, 'epoch': 1.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-33000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-33000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-33000\\special_tokens_map.json\n",
      " 65%|██████▍   | 33500/51747 [3:10:39<1:40:02,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-33500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-33500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3075, 'learning_rate': 7.052389510503025e-06, 'epoch': 1.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-33500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-33500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-33500\\special_tokens_map.json\n",
      " 66%|██████▌   | 34000/51747 [3:13:25<1:36:54,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-34000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-34000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.295, 'learning_rate': 6.859141592749339e-06, 'epoch': 1.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-34000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-34000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-34000\\special_tokens_map.json\n",
      " 67%|██████▋   | 34497/51747 [3:16:11<1:34:29,  3.04it/s]***** Running Evaluation *****\n",
      "  Num examples = 12852\n",
      "  Batch size = 8\n",
      "\n",
      " 67%|██████▋   | 34498/51747 [3:19:48<1:34:28,  3.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5132554173469543, 'eval_runtime': 217.3393, 'eval_samples_per_second': 59.133, 'eval_steps_per_second': 7.394, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 34500/51747 [3:19:49<182:04:51, 38.01s/it]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-34500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-34500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2952, 'learning_rate': 6.665893674995653e-06, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-34500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-34500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-34500\\special_tokens_map.json\n",
      " 68%|██████▊   | 35000/51747 [3:22:35<1:31:53,  3.04it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-35000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-35000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2057, 'learning_rate': 6.472645757241966e-06, 'epoch': 2.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-35000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-35000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-35000\\special_tokens_map.json\n",
      " 69%|██████▊   | 35500/51747 [3:25:22<1:28:49,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-35500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-35500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2201, 'learning_rate': 6.2793978394882805e-06, 'epoch': 2.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-35500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-35500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-35500\\special_tokens_map.json\n",
      " 70%|██████▉   | 36000/51747 [3:28:08<1:26:01,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-36000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-36000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2498, 'learning_rate': 6.086149921734595e-06, 'epoch': 2.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-36000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-36000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-36000\\special_tokens_map.json\n",
      " 71%|███████   | 36500/51747 [3:30:54<1:23:20,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-36500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-36500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2173, 'learning_rate': 5.892902003980907e-06, 'epoch': 2.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-36500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-36500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-36500\\special_tokens_map.json\n",
      " 72%|███████▏  | 37000/51747 [3:33:41<1:20:38,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-37000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-37000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2213, 'learning_rate': 5.699654086227221e-06, 'epoch': 2.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-37000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-37000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-37000\\special_tokens_map.json\n",
      " 72%|███████▏  | 37500/51747 [3:36:27<1:17:50,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-37500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-37500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2269, 'learning_rate': 5.5064061684735345e-06, 'epoch': 2.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-37500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-37500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-37500\\special_tokens_map.json\n",
      " 73%|███████▎  | 38000/51747 [3:39:14<1:15:05,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-38000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-38000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1933, 'learning_rate': 5.313158250719849e-06, 'epoch': 2.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-38000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-38000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-38000\\special_tokens_map.json\n",
      " 74%|███████▍  | 38500/51747 [3:42:00<1:12:31,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-38500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-38500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2266, 'learning_rate': 5.119910332966163e-06, 'epoch': 2.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-38500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-38500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-38500\\special_tokens_map.json\n",
      " 75%|███████▌  | 39000/51747 [3:44:47<1:09:36,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-39000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-39000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2296, 'learning_rate': 4.926662415212477e-06, 'epoch': 2.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-39000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-39000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-39000\\special_tokens_map.json\n",
      " 76%|███████▋  | 39500/51747 [3:47:36<1:07:25,  3.03it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-39500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-39500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2335, 'learning_rate': 4.73341449745879e-06, 'epoch': 2.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-39500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-39500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-39500\\special_tokens_map.json\n",
      " 77%|███████▋  | 40000/51747 [3:50:23<1:04:17,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-40000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-40000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1998, 'learning_rate': 4.540166579705104e-06, 'epoch': 2.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-40000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-40000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-40000\\special_tokens_map.json\n",
      " 78%|███████▊  | 40500/51747 [3:53:10<1:01:36,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-40500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-40500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2089, 'learning_rate': 4.346918661951418e-06, 'epoch': 2.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-40500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-40500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-40500\\special_tokens_map.json\n",
      " 79%|███████▉  | 41000/51747 [3:55:56<58:53,  3.04it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-41000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-41000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1956, 'learning_rate': 4.153670744197732e-06, 'epoch': 2.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-41000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-41000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-41000\\special_tokens_map.json\n",
      " 80%|████████  | 41500/51747 [3:58:43<56:03,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-41500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-41500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2132, 'learning_rate': 3.960422826444045e-06, 'epoch': 2.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-41500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-41500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-41500\\special_tokens_map.json\n",
      " 81%|████████  | 42000/51747 [4:01:29<53:18,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-42000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-42000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2328, 'learning_rate': 3.767174908690359e-06, 'epoch': 2.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-42000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-42000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-42000\\special_tokens_map.json\n",
      " 82%|████████▏ | 42500/51747 [4:04:15<50:32,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-42500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-42500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2084, 'learning_rate': 3.5739269909366732e-06, 'epoch': 2.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-42500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-42500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-42500\\special_tokens_map.json\n",
      " 83%|████████▎ | 43000/51747 [4:07:02<47:46,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-43000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-43000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2211, 'learning_rate': 3.380679073182987e-06, 'epoch': 2.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-43000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-43000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-43000\\special_tokens_map.json\n",
      " 84%|████████▍ | 43500/51747 [4:09:48<44:58,  3.06it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-43500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-43500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2287, 'learning_rate': 3.1874311554293002e-06, 'epoch': 2.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-43500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-43500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-43500\\special_tokens_map.json\n",
      " 85%|████████▌ | 44000/51747 [4:12:35<42:18,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-44000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-44000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2119, 'learning_rate': 2.994183237675614e-06, 'epoch': 2.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-44000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-44000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-44000\\special_tokens_map.json\n",
      " 86%|████████▌ | 44500/51747 [4:15:21<39:42,  3.04it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-44500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-44500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2163, 'learning_rate': 2.800935319921928e-06, 'epoch': 2.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-44500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-44500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-44500\\special_tokens_map.json\n",
      " 87%|████████▋ | 45000/51747 [4:18:08<36:53,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-45000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-45000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.22, 'learning_rate': 2.6076874021682418e-06, 'epoch': 2.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-45000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-45000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-45000\\special_tokens_map.json\n",
      " 88%|████████▊ | 45500/51747 [4:20:55<34:11,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-45500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-45500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2141, 'learning_rate': 2.4144394844145555e-06, 'epoch': 2.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-45500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-45500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-45500\\special_tokens_map.json\n",
      " 89%|████████▉ | 46000/51747 [4:23:41<31:32,  3.04it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-46000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-46000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2246, 'learning_rate': 2.221191566660869e-06, 'epoch': 2.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-46000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-46000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-46000\\special_tokens_map.json\n",
      " 90%|████████▉ | 46500/51747 [4:26:27<28:42,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-46500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-46500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2203, 'learning_rate': 2.0279436489071833e-06, 'epoch': 2.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-46500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-46500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-46500\\special_tokens_map.json\n",
      " 91%|█████████ | 47000/51747 [4:29:13<25:56,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-47000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-47000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2125, 'learning_rate': 1.8346957311534968e-06, 'epoch': 2.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-47000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-47000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-47000\\special_tokens_map.json\n",
      " 92%|█████████▏| 47500/51747 [4:32:01<23:12,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-47500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-47500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1898, 'learning_rate': 1.6414478133998108e-06, 'epoch': 2.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-47500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-47500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-47500\\special_tokens_map.json\n",
      " 93%|█████████▎| 48000/51747 [4:34:47<20:29,  3.05it/s]  Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-48000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-48000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2219, 'learning_rate': 1.4481998956461247e-06, 'epoch': 2.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-48000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-48000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-48000\\special_tokens_map.json\n",
      " 94%|█████████▎| 48500/51747 [4:37:34<17:45,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-48500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-48500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2164, 'learning_rate': 1.2549519778924382e-06, 'epoch': 2.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-48500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-48500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-48500\\special_tokens_map.json\n",
      " 95%|█████████▍| 49000/51747 [4:40:20<15:01,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-49000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-49000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2064, 'learning_rate': 1.061704060138752e-06, 'epoch': 2.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-49000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-49000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-49000\\special_tokens_map.json\n",
      " 96%|█████████▌| 49500/51747 [4:43:07<12:18,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-49500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-49500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2078, 'learning_rate': 8.684561423850658e-07, 'epoch': 2.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-49500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-49500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-49500\\special_tokens_map.json\n",
      " 97%|█████████▋| 50000/51747 [4:45:53<09:32,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-50000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-50000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.191, 'learning_rate': 6.752082246313796e-07, 'epoch': 2.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-50000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-50000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-50000\\special_tokens_map.json\n",
      " 98%|█████████▊| 50500/51747 [4:48:40<06:48,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-50500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-50500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2216, 'learning_rate': 4.819603068776934e-07, 'epoch': 2.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-50500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-50500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-50500\\special_tokens_map.json\n",
      " 99%|█████████▊| 51000/51747 [4:51:26<04:04,  3.05it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-51000\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-51000\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1903, 'learning_rate': 2.887123891240072e-07, 'epoch': 2.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-51000\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-51000\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-51000\\special_tokens_map.json\n",
      "100%|█████████▉| 51500/51747 [4:54:13<01:21,  3.04it/s]Saving model checkpoint to xlnet-base-cased-finetuned-squad\\checkpoint-51500\n",
      "Configuration saved in xlnet-base-cased-finetuned-squad\\checkpoint-51500\\config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2081, 'learning_rate': 9.546447137032099e-08, 'epoch': 2.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in xlnet-base-cased-finetuned-squad\\checkpoint-51500\\pytorch_model.bin\n",
      "tokenizer config file saved in xlnet-base-cased-finetuned-squad\\checkpoint-51500\\tokenizer_config.json\n",
      "Special tokens file saved in xlnet-base-cased-finetuned-squad\\checkpoint-51500\\special_tokens_map.json\n",
      "100%|█████████▉| 51746/51747 [4:55:36<00:00,  3.05it/s]***** Running Evaluation *****\n",
      "  Num examples = 12852\n",
      "  Batch size = 8\n",
      "\n",
      "100%|██████████| 51747/51747 [4:59:13<00:00,  3.05it/s]\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "100%|██████████| 51747/51747 [4:59:13<00:00,  2.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5763306617736816, 'eval_runtime': 217.0592, 'eval_samples_per_second': 59.21, 'eval_steps_per_second': 7.404, 'epoch': 3.0}\n",
      "{'train_runtime': 17953.3942, 'train_samples_per_second': 23.057, 'train_steps_per_second': 2.882, 'train_loss': 0.3545851099190948, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=51747, training_loss=0.3545851099190948, metrics={'train_runtime': 17953.3942, 'train_samples_per_second': 23.057, 'train_steps_per_second': 2.882, 'train_loss': 0.3545851099190948, 'epoch': 3.0})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"google/electra-small-discriminator\".split(\"/\")[-1]\n",
    "\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}-finetuned-squad\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=False,\n",
    "\n",
    ")\n",
    "\n",
    "data_collator = default_data_collator\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to test-squad-trained_xlnet_RETRAINED\n",
      "Configuration saved in test-squad-trained_xlnet_RETRAINED\\config.json\n",
      "Model weights saved in test-squad-trained_xlnet_RETRAINED\\pytorch_model.bin\n",
      "tokenizer config file saved in test-squad-trained_xlnet_RETRAINED\\tokenizer_config.json\n",
      "Special tokens file saved in test-squad-trained_xlnet_RETRAINED\\special_tokens_map.json\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model(\"test-squad-trained_electra\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}