Pytorch Image Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\siggi\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tqdm\\autonotebook.py:17: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from glob import glob\n",
    "from torchvision.datasets import ImageFolder\n",
    "from torch.utils.data import DataLoader, SubsetRandomSampler\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from torchvision import transforms\n",
    "import torch.optim as optim\n",
    "from torch import nn\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "\n",
    "from torch_lr_finder import LRFinder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_all(model_params):\n",
    "    for param in model_params:\n",
    "        param.requires_grad = False\n",
    "        \n",
    "def unfreeze_all(model_params):\n",
    "    for param in model_params:\n",
    "        param.requires_grad = True\n",
    "        \n",
    "def get_trainable(model_params):\n",
    "    return (p for p in model_params if p.requires_grad)\n",
    "\n",
    "def load_image(filename) :\n",
    "    img = Image.open(filename)\n",
    "    img = img.convert('RGB')\n",
    "    return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Resnet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in C:\\Users\\siggi/.cache\\torch\\hub\\pytorch_vision_master\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ResNet(\n",
       "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace=True)\n",
       "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (5): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (fc): Linear(in_features=512, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resnet = torch.hub.load('pytorch/vision', 'resnet34', pretrained=True)\n",
    "resnet.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the data and do preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = glob('./datasets/images/*.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = set()\n",
    "\n",
    "data = []\n",
    "labels = []\n",
    "\n",
    "# Load the images and get the classnames from the image path\n",
    "for image in filenames:\n",
    "    class_name = image.rsplit(\"\\\\\", 1)[1].rsplit('_', 1)[0]\n",
    "    classes.add(class_name)\n",
    "    img = load_image(image)\n",
    "\n",
    "    data.append(img)\n",
    "    labels.append(class_name)\n",
    "\n",
    "# convert classnames to indices\n",
    "class2idx = {cl: idx for idx, cl in enumerate(classes)}        \n",
    "labels = torch.Tensor(list(map(lambda x: class2idx[x], labels))).long()\n",
    "\n",
    "data = list(zip(data, labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PetDataset(Dataset):\n",
    "    \"Dataset to serve individual images to our model\"\n",
    "    \n",
    "    def __init__(self, data, transforms=None):\n",
    "        self.data = data\n",
    "        self.len = len(data)\n",
    "        self.transforms = transforms\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        img, label = self.data[index]\n",
    "        \n",
    "        if self.transforms:\n",
    "            img = self.transforms(img)\n",
    "            \n",
    "        return img, label\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "\n",
    "# Since the data is not split into train and validation datasets we have to \n",
    "# make sure that when splitting between train and val that all classes are represented in both\n",
    "class Databasket():\n",
    "    \"Helper class to ensure equal distribution of classes in both train and validation datasets\"\n",
    "    \n",
    "    def __init__(self, data, num_cl, val_split=0.2, train_transforms=None, val_transforms=None):\n",
    "        class_values = [[] for x in range(num_cl)]\n",
    "        \n",
    "        # create arrays for each class type\n",
    "        for d in data:\n",
    "            class_values[d[1].item()].append(d)\n",
    "            \n",
    "        self.train_data = []\n",
    "        self.val_data = []\n",
    "        \n",
    "        # put (1-val_split) of the images of each class into the train dataset\n",
    "        # and val_split of the images into the validation dataset\n",
    "        for class_dp in class_values:\n",
    "            split_idx = int(len(class_dp)*(1-val_split))\n",
    "            self.train_data += class_dp[:split_idx]\n",
    "            self.val_data += class_dp[split_idx:]\n",
    "            \n",
    "        self.train_ds = PetDataset(self.train_data, transforms=train_transforms)\n",
    "        self.val_ds = PetDataset(self.val_data, transforms=val_transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Apply transformations to the train dataset\n",
    "train_transforms = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "# apply the same transformations to the validation set, with the exception of the\n",
    "# randomized transformation. We want the validation set to be consistent\n",
    "val_transforms = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "\n",
    "])\n",
    "\n",
    "databasket = Databasket(data, len(classes), val_split=0.2, train_transforms=train_transforms, val_transforms=val_transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_types = (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d)\n",
    "\n",
    "class AdaptiveConcatPool2d(nn.Module):\n",
    "    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`.\"\n",
    "    def __init__(self, sz=None):\n",
    "        \"Output will be 2*sz or 2 if sz is None\"\n",
    "        super(AdaptiveConcatPool2d, self).__init__()\n",
    "        self.output_size = sz or 1\n",
    "        self.ap = nn.AdaptiveAvgPool2d(self.output_size)\n",
    "        self.mp = nn.AdaptiveMaxPool2d(self.output_size)\n",
    "\n",
    "    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n",
    "\n",
    "def head_blocks(in_dim, p, out_dim, activation=None):\n",
    "    \"Basic Linear block\"\n",
    "    layers = [\n",
    "        nn.BatchNorm1d(in_dim),\n",
    "        nn.Dropout(p),\n",
    "        nn.Linear(in_dim, out_dim)\n",
    "    ]\n",
    "    \n",
    "    if activation is not None:\n",
    "        layers.append(activation)\n",
    "        \n",
    "    return layers       \n",
    "    \n",
    "def create_head(nf, nc, bn_final=False):\n",
    "    \"Model head that takes in 'nf' features and outputs 'nc' classes\"\n",
    "    pool = AdaptiveConcatPool2d()\n",
    "    layers = [pool, nn.Flatten()]\n",
    "    layers += head_blocks(nf, 0.25, 512, nn.ReLU(inplace=True))\n",
    "    layers += head_blocks(512, 0.5, nc)\n",
    "    \n",
    "    if bn_final:\n",
    "        layers.append(nn.BatchNorm1d(nc, momentum=0.01))\n",
    "    \n",
    "    return nn.Sequential(*layers)\n",
    "    \n",
    "def requires_grad(layer):\n",
    "    \"Determines whether 'layer' requires gradients\"\n",
    "    ps = list(layer.parameters())\n",
    "    if not ps: return None\n",
    "    return ps[0].requires_grad\n",
    "\n",
    "def cnn_model(model, nc, bn_final=False, init=nn.init.kaiming_normal_):\n",
    "    \"Creates a model using a pretrained 'model' and appends a new head to it with 'nc' outputs\"\n",
    "    \n",
    "    # remove dense and freeze everything\n",
    "    body = nn.Sequential(*list(model.children())[:-2])\n",
    "    head = create_head(1024, nc, bn_final)\n",
    "    \n",
    "    model = nn.Sequential(body, head)\n",
    "    \n",
    "    # freeze the resnet34 base of the model\n",
    "    freeze_all(model[0].parameters())\n",
    "    \n",
    "    # initialize the weights of the head\n",
    "    for child in model[1].children():\n",
    "        if isinstance(child, nn.Module) and (not isinstance(child, bn_types)) and requires_grad(child): \n",
    "            init(child.weight)\n",
    "    \n",
    "    return model\n",
    "\n",
    "num_classes = len(classes)\n",
    "model = cnn_model(resnet, num_classes, bn_final=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "    (4): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (7): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): Sequential(\n",
       "    (0): AdaptiveConcatPool2d(\n",
       "      (ap): AdaptiveAvgPool2d(output_size=1)\n",
       "      (mp): AdaptiveMaxPool2d(output_size=1)\n",
       "    )\n",
       "    (1): Flatten()\n",
       "    (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Dropout(p=0.25, inplace=False)\n",
       "    (4): Linear(in_features=1024, out_features=512, bias=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (7): Dropout(p=0.5, inplace=False)\n",
       "    (8): Linear(in_features=512, out_features=37, bias=True)\n",
       "    (9): BatchNorm1d(37, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indices = list(range(len(databasket.train_ds)))\n",
    "test_indices = list(range(len(databasket.val_ds)))\n",
    "\n",
    "train_sampler = SubsetRandomSampler(train_indices)\n",
    "test_sampler = SubsetRandomSampler(test_indices)\n",
    "\n",
    "# Basic dataloader to retrieve mini-batches from the datasets\n",
    "train_loader = DataLoader(databasket.train_ds, batch_size=64, sampler=train_sampler, shuffle=False, num_workers=0)\n",
    "test_loader = DataLoader(databasket.val_ds, batch_size=64, sampler=test_sampler, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "# We don't actually use the learning rate here. It's set to 1e-7 so that the LR Finder\n",
    "# starts at 1e-7\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-7,  weight_decay=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "944dce2855d8461d9e786c5cddc2d607",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3gc9bnF8e+7q95tS+4yrmBk4yob002N4dICxJRAgEtoCS1AAlwCBFJoqUACoZOEFkoAGxwSiCkxxchN7uAG7pZcJKu33/1jF6II2ZZkjWbL+TzPPJqdHe2e0Up7NDM7M+acQ0RE4lfA7wAiIuIvFYGISJxTEYiIxDkVgYhInFMRiIjEORWBiEicS/A7QHvl5ua6gQMH+h1DRCSqzJkzp9Q5l9fafVFXBAMHDqSoqMjvGCIiUcXMPt/Vfdo0JCIS51QEIiJxTkUgIhLnVAQiInFORSAiEudUBCIicS7qPj4q0auqroGVWyrZVlVH76wU+uSkkJWS6HcskbinIohQTU2O6vpGquoaqa5rpKahEYCAgZkB0NjkaGxyNDQ6tlXVsbmshs3lNeyoric9OYGslASyUhMJmFFZ20BFeNheWUdpRS2lFXXU1DfSLS2JHhlJ9EhPoq6xiU1lNWwqr2V7ZR05aYnkZSaTl5lMdmoiCQEjEDASAkZa0n+eIz0pgUAADAOD8up61m2vZv2OatZtr2bllgrW76j+2nJmJCfQKyv0+LkZyfTMTGFgbhqDctMZlJtOn+xUggHr0p+9SLxREXSixiZHTX0jjS0u9mOE3rwNqKxtYHN5LZvLa9i8s4bN5bWU7KxhS3ktpZV1lFXVsaO6nvLqepo6eM2gtKQgVXWNrd6XEDC6pSeRm5FMbkYSeZnJbK+sY/GGckoraklOCNArK4W+2SmM6JtFWXU9JTtrWVVSSXl1PY0uXD7hEtqT7ulJ9MtJpXBgN87Ky2dYrwy6pyezubyGjWXVbNhRQ8nOWkp21rJ4Qzlvl22huv4/2c1CZZGVkkhWaiKD89IZ3T+bA/rlMLJfFplaoxDZayqCPWhscmytrOXzrVV8unknn22uYFVp6E2xqq6BytpGKusaqKprpK6hqd2PbwY90pPIy0whNyOJfbqnkZOWSHZqIhnJCaQlBUlNSiAlMYBhNDlHU7hoguH/zANmdE9PoldWCnmZyaQkBmlsclTUNlBeXY9zkJ4cJD05geSEwFdrFHurpr6R8pp6dtY0UFnbQJMD5xwOyExOoG9OKunJ7fsVc86Fiqe0ktWllWzcUU15TQM7axrYUVXH/C928Hrxxq/m75GeRH73NPK7p9E9LZFgIEAwAInBAEN7ZjCqfzaDcjO0ViGyGyqCFpZuLOeFonV8sLKUkp21bKuqo/k/+GlJQQbnpdMtLYneWSmkJyeQnhwkNSlIWmLoDbvlm45zEHp7hNTEIL2yUr4acjOSSAh2/j77YMDITg0VildSEoOkJAbpmdl5j2lm9MxKoWdWCpMG92h1nq0VtRSvL2PpxnLWbqti7bZqFqzdQXlNPY2NjkbnqG1o+mqNJT0pyJgBORxX0JvjRvSiT3Zq5wUWiQHm9TWLzSwIFAHrnXMntrgvGfgTMB7YCpzpnFuzu8crLCx0HTnXUGlFLUs2lLO9qo7tlXVsq6qnvrGJ5IQASQkBmpocf1+8iUXry0kMGgcPyaVft1RyM5LJy0iif7c0hvXKoG92KgH9dxnxGpscK0sqKF5XxsJ1O5i1cisrtlQAMCY/h5NG9+XUMX3pkZHsc1KRrmFmc5xzha3e1wVFcC1QCGS1UgTfA0Y55y4zs7OAbzrnztzd43W0CKYXb+CKZ+Y1e+7Q9vL6xv8s/4i+WXxrfH9OGdOPbulJ7X4OiWwrtlTw5uJNvLFwI4s3hAr/6OG9OHNCPpP3y+u0TWYikci3IjCz/sBTwM+Ba1spgjeBnzjnPjSzBGATkOd2E6qjRbC1IrTduVtaEt3Tk8hOTSQYMJqaHHWNTdQ3NmnHYxxZtim0CfCVeevZWlnH+H26cdtJBYzqn+N3NBFP+FkELwJ3ApnA9a0UwSJginNuXfj2SuBA51zprh6zo0Ug0pq6hiZembeee95cztbKWs4Y158fTtmPnpkpfkcT6VS7KwLPjiw2sxOBLc65ObubrZVpX2smM7vEzIrMrKikpKTTMookJQSYOiGfmdcfwcWHDeaV+es57O6Z3PhSMcs37fQ7nkiX8GyNwMzuBM4DGoAUIAt42Tl3brN5umzTkEhbrC6t5JH3V/Hy3HXU1Ddx6NBcrjlmGIUDu/sdTWSv+LqzOBxgMq1vGvo+cECzncWnOeem7u6xVATSFbZX1vHsJ1/w5Kw1bNlZy2lj+3Hj8cPpmaVNRhKdfNk0tJswd5jZyeGbjwE9zGwFcC1wY1fnEWlNt/Qkvjd5KO/8cDLfP3II04s3ctSv3uXR91dR39j+AwdFIlmXrBF0Jq0RiB/WlFZy+7TFzFxewvDemfzitAMYN6Cb37FE2iyi1ghEotHA3HSeuHAifzxvPDuq6jn9wQ+4+W8LKauu9zuayF5TEYi0wzdG9Oat647gwoMH8ezsLzj6V+8ybcEGom3NWqQ5FYFIO2UkJ3DrSQW8dsWh9MlO4cpn53H+E5/wxdYqv6OJdIiKQKSDRvbL5pXvH8JtJxUwZ802jvvtu7y2YIPfsUTaTUUgsheCAePCQwbx1nVHMKpfDlc/N48/f/S537FE2kVFINIJ+mSn8qeLJnL08J7c8soi7nv7M+03kKihIhDpJCmJQR48dzynje3Hr//5KbdPW0JTRy8zJ9KFdGEakU6UGAzwy2+NJicticdnrWZ7VR33njGapAT9zyWRS0Ug0skCAeOWE/cnNzOJe/6+nG2VdTx07vh2X7ZTpKvo3xQRD5gZ35s8lHtOH8WsFaWc88hHbK2o9TuWSKtUBCIemjohnz+eV8iyTTs5/4nZVNY2+B1J5GtUBCIeO7agFw+dO56lG3dyxTNzadBJ6yTCqAhEusCRw3vy01NGMnN5Cbe8ukgfLZWIor1XIl3knAMHsH5HFb+fuZL+3dL4/pFD/Y4kAqgIRLrU9cftx7rt1dz75nKG9szgGyN6+x1JRJuGRLqSmXHPGaMY1T+b619YwNptOlGd+E9FINLFkhOC/P6ccQB8/5m51DY0+pxI4p2KQMQH+d3TuPeM0RSvK+PON5b5HUfinIpAxCdTRvbmwkMG8uQHa5ixcKPfcSSOqQhEfHTT8fszOj+HH71YrAvbiG9UBCI+SkoI8MDZY8Hgime1v0D8oSIQ8Vnz/QV3z1judxyJQyoCkQgwZWRvLjh4II/PWs0/Fm/yO47EGRWBSIS46YThjOyXxfUvLGDddu0vkK6jIhCJEMkJQR44exyNTY4fv6LzEUnXURGIRJCBuen84Nh9eWd5CX9fpE1E0jVUBCIR5oKDB7J/nyxun7aECl2/QLqAikAkwiQEA/zs1JFsKq/hd2996ncciQOeFYGZpZjZbDNbYGaLzez2Vua5wMxKzGx+ePiuV3lEosn4fbpx9sR8Hp+1hqUby/2OIzHOyzWCWuAo59xoYAwwxcwmtTLf8865MeHhUQ/ziESVG6YMJzs1kR+/soimJu04Fu94VgQupCJ8MzE86LdZpI1y0pK46fjhzPl8O38tWut3HIlhnu4jMLOgmc0HtgD/dM593Mpsp5tZsZm9aGb5u3icS8ysyMyKSkpKvIwsElHOGN+fAwd1584ZyyitqPU7jsQoT4vAOdfonBsD9AcmmtnIFrNMAwY650YBbwFP7eJxHnbOFTrnCvPy8ryMLBJRzIyff3MkVXUN/OL1pX7HkRjVJZ8acs7tAN4BprSYvtU59+W/OY8A47sij0g0Gdozk0sPH8LL89bzwYpSv+NIDPLyU0N5ZpYTHk8FjgGWtZinT7ObJwP6l0ekFVccNZR9eqTx41cW6Qyl0um8XCPoA8w0s2LgE0L7CKab2R1mdnJ4nqvCHy1dAFwFXOBhHpGolZIY5KenjGRVaSUPvrPS7zgSYyzazmdSWFjoioqK/I4h4osrn53Hm4s28cbVhzG0Z4bfcSSKmNkc51xha/fpyGKRKHLriQWkJgX5v5cX6tgC6TQqApEokpeZzM0n7M/sNdt4XscWSCdREYhEmW8V9mfS4O784o2lbCmv8TuOxAAVgUiUMTN+8c0DqG1o4vZpS/yOIzFARSAShQbnZXDVUUN5feFG3l662e84EuVUBCJR6pLDhzC0ZwY/mbaYmnodWyAdpyIQiVJJCQHuOHkEa7dV89C7OrZAOk5FIBLFDh6ay0mj+/KHd1byxVZd8F46RkUgEuVuPmF/EgPGHdMX+x1FopSKQCTK9c5O4epjhvHW0i28tUQ7jqX9VAQiMeDCQwYxrGcGt09frJPSSbupCERiQGIwwG0nhXYcP/XBGr/jSJRREYjEiEOH5XLkfnnc/68VbK+s8zuORBEVgUgMuemE/amsbeB3b3/mdxSJIioCkRiyb69Mzpo4gL989DmrSyv9jiNRQkUgEmN+cMy+JCcEuHvGsj3PLIKKQCTm5GUmc/nkIfx98SZmr97mdxyJAioCkRh00aGD6Zudwq2vLqKuocnvOBLhVAQiMSg1KchPTx3Jsk07dR4i2SMVgUiMOnr/Xpw0ui8P/GsFn23e6XcciWAqApEYdttJBaQlB7nhpWIadY1j2QUVgUgMy81I5tYTC5j7xQ7+/OEav+NIhFIRiMS4b47tx+H75nHPm8tZt12nqpavUxGIxLjQNY5H4hzc+upinNMmIvlvKgKRONC/WxrXHbcv/1q2henFG/2OIxFGRSASJy48ZBCj+mdz+7TFlFXV+x1HIoiKQCROBAPGnacdwPaqeu6csdTvOBJBPCsCM0sxs9lmtsDMFpvZ7a3Mk2xmz5vZCjP72MwGepVHRGBE32y+e9ggnvtkLR+t2up3HIkQXq4R1AJHOedGA2OAKWY2qcU8FwHbnXNDgd8Ad3uYR0SAa47elwHd0/i/lxfqamYCeFgELqQifDMxPLT8uMIpwFPh8ReBo83MvMokIv85/cSq0koefneV33EkAni6j8DMgmY2H9gC/NM593GLWfoBawGccw1AGdDDy0wiAkfsm8f/HNCHB2au4IutOrYg3nlaBM65RufcGKA/MNHMRraYpbX//r/2IWczu8TMisysqKSkxIuoInHnlhMLSAgYt7y6SMcWxLku+dSQc24H8A4wpcVd64B8ADNLALKBr51A3Tn3sHOu0DlXmJeX53FakfjQOzuFa4/bj3c/LWHGok1+xxEfefmpoTwzywmPpwLHAC0vmfQacH54/AzgX07/moh0mfMP2oeCPlncMW0JFbUNfscRn3i5RtAHmGlmxcAnhPYRTDezO8zs5PA8jwE9zGwFcC1wo4d5RKSFhGCAn39zJJt31nCXji2IWwlePbBzrhgY28r0W5uN1wDf8iqDiOzZ2AHduOiQQTz679UcW9CbI/bV5td4oyOLRYTrv7Efw3pm8KMXF7Cjqs7vONLFVAQiQkpikN+cOYatFXXc8upiv+NIF1MRiAgAI/tlc/XRw5i2YAOvLdjgdxzpQioCEfnK5ZOHMHZADre8soiNZdV+x5EuoiIQka8kBAP8euoY6hqauP6FBTTpOsdxoU1FYGZDzCw5PD7ZzK768hgBEYktg3LTufWkAmat2Mrjs1b7HUe6QFvXCF4CGs1sKKHP/g8CnvEslYj46qwJ+Rxb0It7/r6cJRvK/Y4jHmtrETSFTwr3TeC3zrkfEDpgTERikJlx9+mjyE5L5Jrn51FTr9NVx7K2FkG9mZ1N6HQQ08PTEr2JJCKRoHt6EveeMYpPN1dw75vL/Y4jHmprEVwIHAT83Dm32swGAX/xLpaIRILJ+/Xk3EkDeHzWauav3eF3HPFIm4rAObfEOXeVc+5ZM+sGZDrn7vI4m4hEgBumDKdXZgo3vlRMfWOT33HEA2391NA7ZpZlZt2BBcATZvZrb6OJSCTITEnk9lNGsGzTTh55X1c0i0Vt3TSU7ZwrB04DnnDOjSd0WmkRiQPfGNGbKSN687u3PmNNaaXfcaSTtbUIEsysDzCV/+wsFpE4cvspI0gKBrj5lYW6olmMaWsR3AG8Cax0zn1iZoOBz7yLJSKRpldWCjccP5xZK7by/Cdr/Y4jnaitO4tfcM6Ncs5dHr69yjl3urfRRCTSnDNxAAcN7sFPpy9h7TZd9D5WtHVncX8z+5uZbTGzzWb2kpn19zqciESWQMD45dTRBMy4Tuciihlt3TT0BKHrC/cF+gHTwtNEJM70y0nltpNHMHv1Np2LKEa0tQjynHNPOOcawsOTgK5nJxKnTh/Xj+MKenHPm8v5dPNOv+PIXmprEZSa2blmFgwP5wJbvQwmIpHLzPjFaQeQmZzAD56fT12DDjSLZm0tgv8l9NHRTcBG4AxCp50QkTiVm5HMnacdwOIN5fz2rU/9jiN7oa2fGvrCOXeycy7POdfTOXcqoYPLRCSOHTeiN2dNyOfBd1cye/U2v+NIB+3NFcqu7bQUIhK1bjmxgAHd0/jB8/Mpr6n3O450wN4UgXVaChGJWunJCfzmzDFsKq/hJ68u9juOdMDeFIE+QCwiAIwb0I0rjxrKy/PWM714g99xpJ12WwRmttPMylsZdhI6pkBEBIArjhzKmPwcbv7bIjaV1fgdR9pht0XgnMt0zmW1MmQ65xK6KqSIRL6EYIDfnDmGuoYmrtdRx1FlbzYNiYj8l0G56dxyYgH/XlHKkx+s8TuOtJFnRWBm+WY208yWmtliM7u6lXkmm1mZmc0PD7d6lUdEusbZE/M5enhP7vr7Mj7TUcdRwcs1ggbgOufc/sAk4PtmVtDKfO8758aEhzs8zCMiXcDMuOv0UWQmJ3D1c/OpbWj0O5LsgWdF4Jzb6JybGx7fCSwldMI6EYlxeZnJ3HX6KJZsLOfON5b5HUf2oEv2EZjZQGAs8HErdx9kZgvMbIaZjdjF919iZkVmVlRSUuJhUhHpLMcW9OKiQwfx5AdreL14o99xZDc8LwIzywBeAq4JX/e4ubnAPs650cD9wCutPYZz7mHnXKFzrjAvTyc9FYkWNx4/nHEDcrjhpWJWlVT4HUd2wdMiMLNEQiXwtHPu5Zb3O+fKnXMV4fE3gEQzy/Uyk4h0ncRggAfOGUdi0Pje03OprtP+gkjk5aeGDHgMWOqc+/Uu5ukdng8zmxjOo9Nbi8SQvjmp/ObMMSzfvJOfvKZTUEQiLw8KOwQ4D1hoZvPD0/4PGADgnHuI0OmsLzezBqAaOMs5p6NQRGLM5P16cvkRQ/jDOys5pqAXxxb08juSNGPR9r5bWFjoioqK/I4hIu1U19DEyQ/8m9KKOv7xg8Ppnp7kd6S4YmZznHOFrd2nI4tFpEskJQT49dQxlFXXccuri/yOI82oCESkyxT0zeKaY/bl9eKNTFugs5RGChWBiHSpSw8fzOj8HG55dRFbynWW0kigIhCRLpUQDPDrqaOpqW/kqufm0dCoC9/7TUUgIl1uSF4GPz/1AD5atY173lzud5y4pyIQEV+cPr4/503ah4ffW8UbC3UKCj+pCETEN7ecWMDYATn88IUFrNiiU1b7RUUgIr5JSgjwh2+PIzUpyKV/nkNlbYPfkeKSikBEfNUnO5X7zh7LqtJKbtMpKHyhIhAR3x08JJcrjxzKi3PW8cq89X7HiTsqAhGJCFcdPYwJA7tx898Wsqa00u84cUVFICIRISEY4LdnjSUhGOCq5+ZR16DjC7qKikBEIka/nFTuPn0UxevKuGuGLnHZVVQEIhJRpozszfkH7cPjs1bz6nztL+gKKgIRiTg3/08BEwZ244aXilmyoeUVbqWzqQhEJOIkJQT4/bfHkZ2ayKV/KWJHVZ3fkWKaikBEIlLPzBQePHc8m8tqufLZeTQ2RddFtKKJikBEIta4Ad2445QRvP9ZKXfNWOp3nJjl5TWLRUT22lkTB7BkYzmPvL+aYb0ymVqY73ekmKM1AhGJeLeeWMChQ3O5+W8Lmb16m99xYo6KQEQiXkIwwO/PGUd+tzQu+8sc1m6r8jtSTFERiEhUyE5L5NHzC2lobOK7TxVRXdfod6SYoSIQkagxOC+DB84Zx/LNO7lj+hK/48QMFYGIRJXD983j8slDeHb2F0wv3uB3nJigIhCRqHPtsfsydkAON720kC+2an/B3lIRiEjUSQwGuO+ssWBwpc5UutdUBCISlfK7p3H36aNYsHYHP3t9Cc7pyOOO8qwIzCzfzGaa2VIzW2xmV7cyj5nZfWa2wsyKzWycV3lEJPaccEAfLj5sEH/68HMefm+V33GilpdHFjcA1znn5ppZJjDHzP7pnGu+q/94YFh4OBB4MPxVRKRNbjp+fzaW1XDnjGX0zk7hlDH9/I4UdTxbI3DObXTOzQ2P7wSWAi1foVOAP7mQj4AcM+vjVSYRiT2BgPGrqaM5cFB3rn9hAR+sKPU7UtTpkn0EZjYQGAt83OKufsDaZrfX8fWyEBHZreSEIA9/p5BBuelc+uc5LFpf5nekqOJ5EZhZBvAScI1zruUVJqyVb/naHh8zu8TMisysqKSkxIuYIhLlslMTeep/J5KVmsi5j33M0o26oE1beVoEZpZIqASeds693Mos64DmpxLsD3ztCBHn3MPOuULnXGFeXp43YUUk6vXJTuXZiyeRkhDk3Ec/ZsWWnX5HigpefmrIgMeApc65X+9itteA74Q/PTQJKHPObfQqk4jEvgE90njm4gMxM8555GNWl1b6HSnieblGcAhwHnCUmc0PDyeY2WVmdll4njeAVcAK4BHgex7mEZE4MTgvg2cuPpCGJsd5j33M9kpd6nJ3LNoOwigsLHRFRUV+xxCRKDB/7Q6mPvQhEwd158kLJ5AQjN9jaM1sjnOusLX74venIiIxb0x+Dj87dST/XlHKvW8u9ztOxNKlKkUkpk2dkM/C9WX88b1VjOiXzcmj+/odKeJojUBEYt4tJxYwYWA3fvTiAorX7fA7TsRREYhIzEtKCPCHb4+nR3oy3370Y+Z8ruseN6ciEJG4kJeZzAuXHURuRjLnPTZbp6JoRkUgInGjb04qz186ifxuaVzw5Cf8a9lmvyNFBBWBiMSVnpkpPHfJJIb3zuSSP81hxkIdw6oiEJG40y09ib9890BG5+dwxbPzmLYgvq99rCIQkbiUlRI6Sd34fbpx9XPzeHnuOr8j+UZFICJxKyM5gScvnMBBQ3pw3QsL+Osna/f8TTFIRSAicS0tKYHHzp/AYcPyuOHlYv5aFH9loCIQkbiXkhjk4fPGc+jQXG54qZgX58TXZiIVgYgIoTJ45DuFHDo0lx++uCCu9hmoCEREwr4sg0OG5HLdCwt4bvYXfkfqEioCEZFmviyDI/bN48aXF3L/258Rbafrby8VgYhIC6lJoTI4bVw/fvXPT7nttcU0NsVuGeg01CIirUgMBvjVt0aTl5HMH99bxdbKOn575hgSY/DiNioCEZFdMDNuOmF/emQk8Ys3lhEw4zdTR8fclc5UBCIie3DJ4UMA+MUby0gMGPd+azTBgPmcqvOoCERE2uCSw4dQ19DEL//xKQlB467TRhGIkTJQEYiItNEVRw2jrtFx39ufUVXXyN2njyI9OfrfRqN/CUREutAPjhlGamKQe99cxrJNO3nw2+MY1ivT71h7Jbb2eIiIeMzMuHzyEP5y0YHsqKrj5Adm8er89X7H2isqAhGRDjh4aC6vX3UYI/tlcfVz87lzxlKaovRYAxWBiEgH9cpK4ZmLJ3HupAH88d1VXPaXOVTVNfgdq91UBCIieyExGOCnp4zkJycV8NbSzXzroQ/ZWFbtd6x2URGIiOwlM+OCQwbx2PkT+HxrFWc8+CEbdkRPGagIREQ6yZHDe/LsxZMor67n3Ec/prSi1u9IbeJZEZjZ42a2xcwW7eL+yWZWZmbzw8OtXmUREekqB/TP5okLJ7CxrIbzHptNWVW935H2yMs1gieBKXuY533n3JjwcIeHWUREukzhwO48/J3xrNxSwflPzKa8JrLLwLMicM69B2zz6vFFRCLZYcPyeOCcsSxcX8Y3fvMeby/d7HekXfJ7H8FBZrbAzGaY2YhdzWRml5hZkZkVlZSUdGU+EZEOO25Eb1687CCyUhK56KkirnhmLiU7I2+/gXl55R0zGwhMd86NbOW+LKDJOVdhZicAv3PODdvTYxYWFrqioqJOzyoi4pW6hib++O5K7v/XCpISAkwtzOeCgwcyoEdal2UwsznOucLW7vNtjcA5V+6cqwiPvwEkmlmuX3lERLySlBDgyqOH8cbVh3HU8J786cM1HPHLmXz3qSLmfO7/FnTfisDMepuZhccnhrNs9SuPiIjXhvbM4L6zx/LvG47i+5OHMveL7Zz+4Idc8qciVmyp8C2XZ5uGzOxZYDKQC2wGbgMSAZxzD5nZFcDlQANQDVzrnPtgT4+rTUMiEiuq6hp47P3V/PG9VVTXN3LWhHx+NGU42amJnf5cu9s05Ok+Ai+oCEQk1pRW1HL/25/x9Mdf0L9bKg+dN57hvbM69Tkich+BiIiE5GYkc/spI3nukklU1TVy6u+79tTWWiMQEYkgW3bWcMXT85i9ZhtH7JtHamKQ2oZGahuaOHFUX845cECHHldrBCIiUaJnZgpPX3wglx4xmC+2VbG6tJKtlXXUNzbh8GifrtYIRERin9YIRERkl1QEIiJxTkUgIhLnVAQiInFORSAiEudUBCIicU5FICIS51QEIiJxLuoOKDOzEmAHUNbiruwW05rfbm38y6+5QGkH47R8zvbMs7u8LW/vabyjy9CZ+VubFgmvgZf5m49H4mvQluWJ9teg+bRoeg0683dod/ma35/jnMtr9V7nXNQNwMN7mtb8dmvjzb4WdWaOts6zu7xtyd8Zy9CZ+SP1NfAyf6S/Bm1Znmh/DVpMi5rXoDN/h/bmNfhyiNZNQ9PaMG3aHsZbe4zOyNHWeXaXt+Xttox3RGfmb21aJLwGXuZvy/PviZevQVuWJ9pfg0j7O245bVfL05m/Q215jN3eH3WbhjqbmRW5XZx/I1pE+zJEe36I/mWI9vwQ/VSobLYAAAYZSURBVMvgZ/5oXSPoTA/7HaATRPsyRHt+iP5liPb8EP3L4Fv+uF8jEBGJd1ojEBGJcyoCEZE4pyIQEYlzKoLdMLOAmf3czO43s/P9ztMRZjbZzN43s4fMbLLfeTrCzNLNbI6Zneh3lvYys/3DP/sXzexyv/N0hJmdamaPmNmrZnac33nay8wGm9ljZvai31naI/x7/1T4Z/9tL58rZovAzB43sy1mtqjF9ClmttzMVpjZjXt4mFOAfkA9sM6rrLvSScvggAoghS5ehk7KD3AD8FdvUu5aZ+R3zi11zl0GTAW6/KOBnbQMrzjnLgYuAM70MO7XdFL+Vc65i7xN2jbtXJ7TgBfDP/uTPQ3W0SPZIn0ADgfGAYuaTQsCK4HBQBKwACgADgCmtxh6AjcCl4a/98UoXYZA+Pt6AU9HYf5jgLMIvQmdGG35w99zMvABcE40/g41+75fAeOiOH+X/w3v5fLcBIwJz/OMl7kSiFHOuffMbGCLyROBFc65VQBm9hxwinPuTuBrmx3MbB1QF77Z6F3a1nXGMjSzHUj2IueudNJrcCSQTugPo9rM3nDONXkaPKyzfv7OudeA18zsdeAZ7xK3+tyd8RoYcBcwwzk319vE/62T/wZ8157lIbQG3x+Yj8dbb2K2CHahH7C22e11wIG7mf9l4H4zOwx4z8tg7dCuZTCz04BvADnAA95Ga5N25XfO3QxgZhcApV1VArvR3p//ZEKr+MnAG54ma7v2/h1cSWjNLNvMhjrnHvIyXBu09zXoAfwcGGtmN4ULI5LsannuAx4ws/+hc05DsUvxVgTWyrRdHlHnnKsCImLbYjPtXYaXCRVapGhX/q9mcO7Jzo/SIe39+b8DvONVmA5q7zLcR+hNKVK0N/9W4DLv4uy1VpfHOVcJXNgVAWJ2Z/EurAPym93uD2zwKUtHRfsyKL//on0Zoj1/S74vT7wVwSfAMDMbZGZJhHZCvuZzpvaK9mVQfv9F+zJEe/6W/F8ev/eie7h3/llgI//56OdF4eknAJ8S2kt/s985Y3kZlN//IdqXIdrzR8vy6KRzIiJxLt42DYmISAsqAhGROKciEBGJcyoCEZE4pyIQEYlzKgIRkTinIpCYYWYVXfx8j5pZQRc/5zVmltaVzymxT8cRSMwwswrnXEYnPl6Cc66hsx6vjc9phP4uWz25npmtAQqdc6VdmUtim9YIJKaZWZ6ZvWRmn4SHQ8LTJ5rZB2Y2L/x1v/D0C8zsBTObBvzDQld4e8dCVxhbZmZPh9+sCU8vDI9XWOhqdgvM7CMz6xWePiR8+xMzu6O1tRYzG2hmS83sD8BcIN/MHjSzIjNbbGa3h+e7CugLzDSzmeFpx5nZh2Y2N5y704pQ4ojfh1xr0NBZA1DRyrRngEPD4wOApeHxLCAhPH4M8FJ4/AJCh/53D9+eDJQROhFYAPiw2eO9Q+i/cwid/fKk8Pg9wI/D49OBs8Pjl+0i40CgCZjUbNqXzx8MP8+o8O01QG54PJfQ6dHTw7dvAG71+3XQEH1DvJ2GWuLPMUBB+J94gCwzywSygafMbBihN/HEZt/zT+fctma3Zzvn1gGY2XxCb9z/bvE8dYTe9AHmAMeGxw8CTg2PPwP8chc5P3fOfdTs9lQzu4TQqeL7ELowT3GL75kUnj4rvHxJhIpKpF1UBBLrAsBBzrnq5hPN7H5gpnPum+ErRr3T7O7KFo9R22y8kdb/buqdc24P8+zOV89pZoOA64EJzrntZvYkoWtOt2SESuvsdj6XyH/RPgKJdf8ArvjyhpmNCY9mA+vD4xd4+PwfAaeHx89q4/dkESqGsvC+huOb3bcTyGz22IeY2VAAM0szs333PrLEGxWBxJI0M1vXbLgWuAooNLNiM1vCf65UdQ9wp5nNIrQd3ivXANea2WxCm3jK9vQNzrkFwDxgMfA4MKvZ3Q8DM8xspnOuhFCJPWtmxYSKYXjnxpd4oI+Pingo/Jn/auecM7OzCO04PsXvXCLNaR+BiLfGE7oAuQE7gP/1OY/I12iNQEQkzmkfgYhInFMRiIjEORWBiEicUxGIiMQ5FYGISJxTEYiIxLn/Bw8eCL1O4HIbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# used to find the ideal LR for our model training\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device=\"cuda\")\n",
    "lr_finder.range_test(train_loader, end_lr=3, num_iter=100)\n",
    "lr_finder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A one cycle LR scheduler (https://arxiv.org/abs/1708.07120)\n",
    "# max_lr is derived from the lr_finder plot\n",
    "# epochs must match the amount of epochs you will run\n",
    "scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=5e-2, pct_start=0.3, steps_per_epoch=len(train_loader), epochs=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: Train Accuracy: 48.66 percent, Validation Accuracy: 72.75 percent, Train Loss: 2.8240740670952746, Validation Loss: 0.14077554643154144\n",
      "Epoch 1: Train Accuracy: 62.61 percent, Validation Accuracy: 87.96 percent, Train Loss: 1.645881425949835, Validation Loss: 0.02494307979941368\n",
      "Epoch 2: Train Accuracy: 75.35 percent, Validation Accuracy: 90.67 percent, Train Loss: 0.889891113645287, Validation Loss: 0.205683633685112\n",
      "Epoch 3: Train Accuracy: 79.95 percent, Validation Accuracy: 92.29 percent, Train Loss: 0.6370588678826568, Validation Loss: 0.012420245446264744\n",
      "Epoch 4: Train Accuracy: 82.91 percent, Validation Accuracy: 92.36 percent, Train Loss: 0.5365907344766843, Validation Loss: 0.017855508252978325\n",
      "Epoch 5: Train Accuracy: 85.55 percent, Validation Accuracy: 92.56 percent, Train Loss: 0.45028836816869755, Validation Loss: 0.4628230631351471\n",
      "Epoch 6: Train Accuracy: 86.30 percent, Validation Accuracy: 92.97 percent, Train Loss: 0.4157002092369141, Validation Loss: 0.011852264404296875\n",
      "Epoch 7: Train Accuracy: 87.18 percent, Validation Accuracy: 92.63 percent, Train Loss: 0.3850554785420818, Validation Loss: 0.1731860339641571\n",
      "Epoch 8: Train Accuracy: 88.39 percent, Validation Accuracy: 93.17 percent, Train Loss: 0.3439134707053502, Validation Loss: 0.48683643341064453\n",
      "Epoch 9: Train Accuracy: 88.78 percent, Validation Accuracy: 93.17 percent, Train Loss: 0.3364143895526086, Validation Loss: 0.0016621181275695562\n",
      "Epoch 10: Train Accuracy: 89.75 percent, Validation Accuracy: 93.31 percent, Train Loss: 0.29718081332663054, Validation Loss: 0.06409672647714615\n",
      "Epoch 11: Train Accuracy: 90.29 percent, Validation Accuracy: 93.17 percent, Train Loss: 0.28224363923072815, Validation Loss: 0.008844920434057713\n",
      "Epoch 12: Train Accuracy: 90.07 percent, Validation Accuracy: 93.51 percent, Train Loss: 0.28206250976811176, Validation Loss: 0.008843149058520794\n",
      "Epoch 13: Train Accuracy: 90.53 percent, Validation Accuracy: 93.37 percent, Train Loss: 0.263952146774979, Validation Loss: 0.1329212188720703\n",
      "Epoch 14: Train Accuracy: 90.36 percent, Validation Accuracy: 93.44 percent, Train Loss: 0.2793284943026881, Validation Loss: 0.027688708156347275\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda:0'\n",
    "model.to(device)\n",
    "\n",
    "def train(epochs, scheduler, optimizer, model):\n",
    "    for epoch in range(epochs):\n",
    "        running_loss = 0\n",
    "        n_correct = 0\n",
    "        \n",
    "        # use dropouts and batchnorms\n",
    "        model.train()\n",
    "        for batch in train_loader:\n",
    "            inputs, labels = batch\n",
    "            \n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "            \n",
    "            #zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            outputs = model(inputs)\n",
    "            \n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            \n",
    "            optimizer.step()\n",
    "            scheduler.step()\n",
    "            \n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            n_correct += (predicted == labels).sum().item()\n",
    "            \n",
    "            running_loss += loss.item()\n",
    "        \n",
    "        train_acc = 100. * n_correct / len(databasket.train_ds)\n",
    "        train_loss = running_loss / len(train_loader)\n",
    "        \n",
    "        n_val_correct = 0\n",
    "        val_loss = 0\n",
    "        \n",
    "        # disable batchnorm and dropouts\n",
    "        model.eval()\n",
    "        # don't calculate gradient\n",
    "        with torch.no_grad():\n",
    "            for batch in test_loader:\n",
    "                inputs, labels = batch\n",
    "                \n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                outputs = model(inputs)\n",
    "                val_loss = criterion(outputs, labels).item()\n",
    "\n",
    "                n_val_correct += (torch.max(outputs.data, 1)[1] == labels).sum().item()\n",
    "                \n",
    "                \n",
    "                \n",
    "        val_acc = 100. * n_val_correct / len(databasket.val_ds)\n",
    "                \n",
    "                \n",
    "                \n",
    "            \n",
    "        print('Epoch %s: Train Accuracy: %.2f percent, Validation Accuracy: %.2f percent, Train Loss: %s, Validation Loss: %s' \n",
    "              % (epoch, train_acc, val_acc, train_loss, val_loss))\n",
    "        \n",
    "train(15, scheduler, optimizer, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save({\n",
    "            'epoch': 15,\n",
    "            'model_state_dict': model.state_dict(),\n",
    "            'optimizer_state_dict': optimizer.state_dict(),\n",
    "            }, './models/oxford_animal_frozen_1.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "unfreeze_all(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: Train Accuracy: 91.52 percent, Validation Accuracy: 93.10 percent, Train Loss: 0.2606610147863306, Validation Loss: 0.03882421925663948\n",
      "Epoch 1: Train Accuracy: 91.00 percent, Validation Accuracy: 93.44 percent, Train Loss: 0.26458921499790683, Validation Loss: 0.2725500464439392\n",
      "Epoch 2: Train Accuracy: 91.00 percent, Validation Accuracy: 93.64 percent, Train Loss: 0.2578640261324503, Validation Loss: 0.10353279113769531\n",
      "Epoch 3: Train Accuracy: 91.59 percent, Validation Accuracy: 93.51 percent, Train Loss: 0.2522153679722099, Validation Loss: 0.0198211669921875\n",
      "Epoch 4: Train Accuracy: 92.12 percent, Validation Accuracy: 93.58 percent, Train Loss: 0.24540977388299923, Validation Loss: 0.41530942916870117\n"
     ]
    }
   ],
   "source": [
    "# discriminative learning rate for fine tuning\n",
    "optimizer = optim.Adam([\n",
    "        {\"params\": model[0].parameters(), \"lr\": 1e-7},\n",
    "        {\"params\": model[1].parameters(), \"lr\": 1e-7}\n",
    "    ],  weight_decay=1e-5)\n",
    "scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=(1e-6,1e-3), pct_start=0.8, steps_per_epoch=len(train_loader), epochs=5)\n",
    "train(5, scheduler, optimizer, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}