04_RNN_LSTM/04_LSTM_IMDB.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "04_LSTM_IMDB.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/satyajitghana/TSAI-DeepNLP-END2.0/blob/main/04_RNN_LSTM/04_LSTM_IMDB.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4trWBrarVO9N",
        "outputId": "1ebd0369-a251-45eb-cbf9-0131e37db363"
      },
      "source": [
        "! nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed May 26 17:34:46 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   52C    P8    10W /  70W |      0MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SPhj6gnAnT2"
      },
      "source": [
        "import torch\n",
        "from torchtext.legacy import data\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "TEXT = data.Field(tokenize = 'spacy',\n",
        "                  tokenizer_language = 'en_core_web_sm',\n",
        "                  include_lengths = True)\n",
        "\n",
        "LABEL = data.LabelField(dtype = torch.float)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lwn4oStE6PzV",
        "outputId": "6c010554-1463-48ce-c528-4d76011d5274"
      },
      "source": [
        "from torchtext.legacy import datasets\n",
        "\n",
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading aclImdb_v1.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:02<00:00, 29.9MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5DLJ86m56Xdn",
        "outputId": "a11dca14-ff00-4613-b4d8-9e13e8bad2be"
      },
      "source": [
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples: 25000\n",
            "Number of testing examples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iXTWwqXA6rP2",
        "outputId": "6c3e4f43-2ba3-4aba-d5d7-a02b522741e0"
      },
      "source": [
        "print(vars(train_data.examples[0]))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': ['Before', 'Cujo', ',', 'there', 'was', 'Lucky', 'the', 'devil', 'dog', '.', 'In', '1978,on', 'Halloween', 'night', 'the', 'movie\"Devil', 'Dog', ',', 'The', 'Hound', 'of', 'Hell', '\"', 'premiered', '.', 'A', 'story', 'of', 'a', 'family', 'getting', 'a', 'new', 'puppy', '(', 'from', 'a', 'farmer', 'who', 'just', 'happen', 'to', 'be', 'in', 'the', 'neighborhood', 'selling', 'fruits', 'and', 'vegetables', ')', 'because', 'their', 'dog', 'Skipper', 'was', 'killed', '.', 'Coencidence', '?', 'Everyone', 'loves', 'the', 'new', 'dog', ',', 'but', 'there', 'is', 'something', 'strange', 'about', 'him', '.', '<', 'br', '/><br', '/>It', 'is', \"n't\", 'long', 'until', 'the', 'father', 'Mike', 'Barry(Richard', 'Crenna', ',', 'First', 'Blood)starts', 'to', 'notice', '.', 'His', 'wife', 'Betty(Yvette', 'Mimieux', ',', 'Where', 'The', 'Boys', 'Are', ',', 'Jackson', 'County', 'Jail', ',', 'Snowbeast)is', 'different', 'and', 'his', 'kids', 'Charlie', 'and', 'Bonnie(Ike', 'Eisenman', ',', 'Witch', 'Mountain', 'and', 'Fantastic', 'Vourage', 'and', 'Kim', 'Richards', ',', 'Witch', 'Mountain', ',', 'Nanny', 'and', 'the', 'Professor', ',', 'Hello', 'Larry', ',', 'Tuff', '-', 'Turf)also', 'have', 'changed', '.', 'Does', 'the', 'dog', 'have', 'something', 'to', 'do', 'with', 'it', '?', 'He', \"'s\", 'determined', 'to', 'find', 'out', 'and', 'do', 'whatever', 'it', 'takes', 'to', 'save', 'his', 'family.<br', '/><br', '/>This', 'movie', 'is', 'great', 'because', 'it', 'has', 'Ike', 'and', 'Kim', 'playing', 'a', 'darker', 'side', 'of', 'themselves', 'than', 'what', 'we', 'saw', 'on', 'those', 'witch', 'mountain', 'movies', '.', 'This', 'is', 'one', 'of', 'the', 'many', '70', \"'s\", 'made', '-', 'for', '-', 'TV', 'horror', 'movies', 'that', 'was', 'actually', 'scary', 'for', 'a', 'made', '-', 'for', '-', 'TV', 'horror', 'movie', '.', 'The', 'music', 'was', 'creepy', 'and', 'even', 'the', 'ending', 'which', 'I', 'wo', \"n't\", 'tell', 'made', 'you', 'think.<br', '/><br', '/>This', 'movie', 'also', 'stars', 'Ken', 'Kercheval(Cliff', 'Barnes', 'of', 'Dallas)and', 'R.G.', 'Armstrong(who', 'could', \"n't\", 'stay', 'away', 'from', 'devil', 'movies', 'remember\"Race', 'with', 'the', 'Devil\"?)<br', '/><br', '/>It', \"'s\", 'worth', 'watching', '.'], 'label': 'pos'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HMVqiZd6tR0"
      },
      "source": [
        "import random\n",
        "\n",
        "train_data, valid_data = train_data.split(random_state = random.seed(SEED))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uOeQ6KpP7M-0",
        "outputId": "40fdf0bd-ecde-4b64-9114-88b3a829dd8b"
      },
      "source": [
        "print(f'Number of training examples: {len(train_data)}')\n",
        "print(f'Number of validation examples: {len(valid_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples: 17500\n",
            "Number of validation examples: 7500\n",
            "Number of testing examples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KixkM1jQ7TB-"
      },
      "source": [
        "MAX_VOCAB_SIZE = 25_000\n",
        "\n",
        "TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hD4SFKnc7g0D",
        "outputId": "f029f071-f012-41f5-8390-d1cc6141df86"
      },
      "source": [
        "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n",
        "print(f\"Unique tokens in LABEL vocabulary: {len(LABEL.vocab)}\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique tokens in TEXT vocabulary: 25002\n",
            "Unique tokens in LABEL vocabulary: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ttKvFTCQ7isK",
        "outputId": "b52fb395-5148-417f-e3c7-1abec226ea05"
      },
      "source": [
        "print(TEXT.vocab.freqs.most_common(20))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('the', 203063), (',', 192343), ('.', 166129), ('and', 109794), ('a', 109526), ('of', 100808), ('to', 93971), ('is', 76428), ('in', 61581), ('I', 54306), ('it', 53609), ('that', 49177), ('\"', 44610), (\"'s\", 43276), ('this', 42373), ('-', 36806), ('/><br', 35659), ('was', 35096), ('as', 30590), ('with', 30113)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fZXIsIV47mlI",
        "outputId": "09c18a61-872e-4956-9cd4-06ba1c5c128e"
      },
      "source": [
        "print(TEXT.vocab.itos[:10])"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<unk>', '<pad>', 'the', ',', '.', 'and', 'a', 'of', 'to', 'is']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vmbx3T9-7x4g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd4447e5-5654-46d5-dfd1-073cdee3acb4"
      },
      "source": [
        "print(LABEL.vocab.stoi)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(None, {'neg': 0, 'pos': 1})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3gBfP6mEJ_0"
      },
      "source": [
        "BATCH_SIZE = 128\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    sort_within_batch=True, # necessary for packed_padded_sequence\n",
        "    device = device)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpaWK7HQ4wiA",
        "outputId": "bb2ab651-1bb4-4d6b-8004-2cddfb33782f"
      },
      "source": [
        "print('Train')\n",
        "for batch in train_iterator:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break\n",
        "    \n",
        "print('\\nValid:')\n",
        "for batch in valid_iterator:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break\n",
        "    \n",
        "print('\\nTest:')\n",
        "for batch in test_iterator:\n",
        "    print(f'Text matrix size: {batch.text[0].size()}')\n",
        "    print(f'Target vector size: {batch.label.size()}')\n",
        "    break"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train\n",
            "Text matrix size: torch.Size([133, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Valid:\n",
            "Text matrix size: torch.Size([55, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Test:\n",
            "Text matrix size: torch.Size([42, 128])\n",
            "Target vector size: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f5mSNdbyYKpt"
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class LSTM(nn.Module):\n",
        "    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim)\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "        \n",
        "    def forward(self, text, text_length):\n",
        "\n",
        "        #[sentence len, batch size] => [sentence len, batch size, embedding size]\n",
        "        embedded = self.embedding(text)\n",
        "        \n",
        "        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, text_length.to('cpu'))\n",
        "        \n",
        "        #[sentence len, batch size, embedding size] => \n",
        "        #  output: [sentence len, batch size, hidden size]\n",
        "        #  hidden: [1, batch size, hidden size]\n",
        "        packed_output, (hidden, cell) = self.lstm(packed)\n",
        "        \n",
        "        return self.fc(hidden.squeeze(0)).view(-1)\n"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x0_X5kSwENad"
      },
      "source": [
        "INPUT_DIM = len(TEXT.vocab)\n",
        "EMBEDDING_DIM = 128\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1\n",
        "\n",
        "model = LSTM(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdGb8dKBEO2x",
        "outputId": "d013baf8-2d18-44ca-8ec0-4c4766713bc5"
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 3,595,777 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAeEtXiJEQCj"
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=1e-4)\n",
        "# NOTE: DO NOT USE SGD HERE, ONLY ADAM"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Utp4-qAERRG"
      },
      "source": [
        "criterion = nn.BCEWithLogitsLoss()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyAXf58FESdL"
      },
      "source": [
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4yNiGXQETh9"
      },
      "source": [
        "def binary_accuracy(preds, y):\n",
        "    \"\"\"\n",
        "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
        "    \"\"\"\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1iGJW1wEUrL"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "\n",
        "        text, text_length = batch.text\n",
        "        logits = model(text, text_length)\n",
        "        \n",
        "        loss = criterion(logits, batch.label)\n",
        "        \n",
        "        acc = binary_accuracy(logits, batch.label)\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNQxQS3tEWUW"
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text, text_length = batch.text\n",
        "            predictions = model(text, text_length)\n",
        "            \n",
        "            loss = criterion(predictions, batch.label)\n",
        "            \n",
        "            acc = binary_accuracy(predictions, batch.label)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DVM8MtV6EYIw"
      },
      "source": [
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJ5KZmM4EZXW",
        "outputId": "f193b179-58e8-434e-fe38-109fb3ab6db6"
      },
      "source": [
        "N_EPOCHS = 30\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "    \n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'tut1-model.pt')\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.687 | Train Acc: 55.23%\n",
            "\t Val. Loss: 0.685 |  Val. Acc: 55.95%\n",
            "Epoch: 02 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.655 | Train Acc: 61.87%\n",
            "\t Val. Loss: 0.605 |  Val. Acc: 67.61%\n",
            "Epoch: 03 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.606 | Train Acc: 68.27%\n",
            "\t Val. Loss: 0.578 |  Val. Acc: 70.49%\n",
            "Epoch: 04 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.544 | Train Acc: 73.28%\n",
            "\t Val. Loss: 0.542 |  Val. Acc: 73.38%\n",
            "Epoch: 05 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.510 | Train Acc: 75.84%\n",
            "\t Val. Loss: 0.518 |  Val. Acc: 74.89%\n",
            "Epoch: 06 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.473 | Train Acc: 78.06%\n",
            "\t Val. Loss: 0.516 |  Val. Acc: 75.59%\n",
            "Epoch: 07 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.442 | Train Acc: 80.16%\n",
            "\t Val. Loss: 0.473 |  Val. Acc: 78.18%\n",
            "Epoch: 08 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.416 | Train Acc: 81.96%\n",
            "\t Val. Loss: 0.449 |  Val. Acc: 79.42%\n",
            "Epoch: 09 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.402 | Train Acc: 82.81%\n",
            "\t Val. Loss: 0.438 |  Val. Acc: 80.90%\n",
            "Epoch: 10 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.377 | Train Acc: 83.86%\n",
            "\t Val. Loss: 0.427 |  Val. Acc: 80.83%\n",
            "Epoch: 11 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.363 | Train Acc: 84.97%\n",
            "\t Val. Loss: 0.426 |  Val. Acc: 81.78%\n",
            "Epoch: 12 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.343 | Train Acc: 85.76%\n",
            "\t Val. Loss: 0.407 |  Val. Acc: 83.05%\n",
            "Epoch: 13 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.361 | Train Acc: 84.98%\n",
            "\t Val. Loss: 0.414 |  Val. Acc: 81.41%\n",
            "Epoch: 14 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.314 | Train Acc: 87.14%\n",
            "\t Val. Loss: 0.408 |  Val. Acc: 82.58%\n",
            "Epoch: 15 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.305 | Train Acc: 87.60%\n",
            "\t Val. Loss: 0.394 |  Val. Acc: 82.71%\n",
            "Epoch: 16 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.289 | Train Acc: 88.66%\n",
            "\t Val. Loss: 0.375 |  Val. Acc: 84.15%\n",
            "Epoch: 17 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.279 | Train Acc: 88.81%\n",
            "\t Val. Loss: 0.388 |  Val. Acc: 84.46%\n",
            "Epoch: 18 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.261 | Train Acc: 89.79%\n",
            "\t Val. Loss: 0.535 |  Val. Acc: 75.68%\n",
            "Epoch: 19 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.258 | Train Acc: 89.81%\n",
            "\t Val. Loss: 0.390 |  Val. Acc: 83.68%\n",
            "Epoch: 20 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.376 | Train Acc: 83.77%\n",
            "\t Val. Loss: 0.398 |  Val. Acc: 84.10%\n",
            "Epoch: 21 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.233 | Train Acc: 91.05%\n",
            "\t Val. Loss: 0.378 |  Val. Acc: 85.10%\n",
            "Epoch: 22 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.215 | Train Acc: 91.93%\n",
            "\t Val. Loss: 0.378 |  Val. Acc: 85.22%\n",
            "Epoch: 23 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.212 | Train Acc: 92.10%\n",
            "\t Val. Loss: 0.364 |  Val. Acc: 85.56%\n",
            "Epoch: 24 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.219 | Train Acc: 91.79%\n",
            "\t Val. Loss: 0.378 |  Val. Acc: 84.86%\n",
            "Epoch: 25 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.195 | Train Acc: 92.93%\n",
            "\t Val. Loss: 0.368 |  Val. Acc: 85.28%\n",
            "Epoch: 26 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.197 | Train Acc: 92.68%\n",
            "\t Val. Loss: 0.375 |  Val. Acc: 83.41%\n",
            "Epoch: 27 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.195 | Train Acc: 92.68%\n",
            "\t Val. Loss: 0.363 |  Val. Acc: 85.24%\n",
            "Epoch: 28 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.189 | Train Acc: 92.96%\n",
            "\t Val. Loss: 0.367 |  Val. Acc: 85.72%\n",
            "Epoch: 29 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.161 | Train Acc: 94.18%\n",
            "\t Val. Loss: 0.394 |  Val. Acc: 84.96%\n",
            "Epoch: 30 | Epoch Time: 0m 7s\n",
            "\tTrain Loss: 0.159 | Train Acc: 94.31%\n",
            "\t Val. Loss: 0.396 |  Val. Acc: 84.50%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qIiKAJMaEbKO",
        "outputId": "646bedb9-6e66-48a2-f54b-e91780e42fa6"
      },
      "source": [
        "model.load_state_dict(torch.load('tut1-model.pt'))\n",
        "\n",
        "test_loss, test_acc = evaluate(model, test_iterator, criterion)\n",
        "\n",
        "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.365 | Test Acc: 85.35%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G024NssCEcj0"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}