breakhisv1-resnetunet-binary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, make some essential imports and then have a look at the data directory. The cell below gives a peek into what is in the BreakHis dataset. The goal is to initially create a pandas dataframe with two columns: images and labels. In this dataset, the file name itself contain the labels, so we will use that information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        pass\n",
    "        # print(os.path.join(dirname, filename))\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "timestamp_exec_start = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just the filenames are enough to get class information of the images. There are four classes for benign, and four for malignant. Ignoring the magnification, there is a total of 8 classes. The next cell simply makes a list of all the files that are images. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed  7909 files\n"
     ]
    }
   ],
   "source": [
    "files = []\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        if (filename[-3:] == 'png'):\n",
    "            files.append(os.path.join(dirname, filename))\n",
    "print(\"Processed \",len(files),\"files\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we build the class labels from the file names and then build up a dataframe. Based on the value of the \"REDUCED_CLASSES\" flag, we either take 8 classes, or we take 2 classes. \n",
    "- B  = Benign\n",
    "    * A = Adenosis\n",
    "    * F = Fibroadenoma\n",
    "    * TA = Tubular Adenoma\n",
    "    * PT = Phyllodes Tumor\n",
    "- M  = Malignant\n",
    "    * DC = Ductal Carcinoma\n",
    "    * LC = Lobular Carcinoma\n",
    "    * MC = Mucinous Carcinoma (Colloid)\n",
    "    * PC = Papillary Carcinoma\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "labels\n",
       "0    2480\n",
       "1    5429\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_dict = {\"B_A-\":0,\"B_F-\":1,\"B_TA\":2,\"B_PT\":3,\"M_DC\":4,\"M_LC\":5,\"M_MC\":6,\"M_PC\":7}  # for 8 class problem\n",
    "labels_dict_simple = {\"B\":0,\"M\":1}                                                       # for 2 class problem\n",
    "REDUCED_CLASSES = True\n",
    "\n",
    "X = []\n",
    "Y = []\n",
    "for f in files:\n",
    "    x = f.split(\"/\") # break up the path\n",
    "    x = x[-1:][0]    # extract the file name\n",
    "    X.append(str(f))\n",
    "    if REDUCED_CLASSES:\n",
    "        Y.append(int(labels_dict_simple[x[4]]))\n",
    "    else:\n",
    "        Y.append(int(labels_dict[x[4:8]]))\n",
    "\n",
    "data = {\"images\":X,\"labels\":Y}\n",
    "images_df = pd.DataFrame(data, columns = ['images','labels'])\n",
    "images_df.groupby(\"labels\")[\"labels\"].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Pytorch Stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6327, 1582)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train, val = train_test_split(images_df, stratify=images_df.labels, test_size=0.2)\n",
    "len(train), len(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, df_data,transform=None):\n",
    "        super().__init__()\n",
    "        self.df = df_data.values\n",
    "        \n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        img_path,label = self.df[index]\n",
    "        \n",
    "        image = cv2.imread(img_path)\n",
    "        image = cv2.resize(image, (224,224))\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Parameters for model\n",
    "\n",
    "# Hyper parameters\n",
    "num_epochs = 50\n",
    "num_classes = 8\n",
    "batch_size = 16\n",
    "learning_rate = 0.0002\n",
    "\n",
    "# Device configuration\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utility function for saving model\n",
    "# During training, the loss values are stored in a list.\n",
    "# We check the last two values to see if the loss has reduced.\n",
    "def save_checkpoint(state, loss):\n",
    "    global best_loss\n",
    "    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n",
    "    if best_loss>=loss:        \n",
    "        print (\"=> Loss reduced by:\\t\",best_loss - loss)\n",
    "        print(\"   Saving model state\")\n",
    "        torch.save(state, \"state_dict.dct\")  # save checkpoint\n",
    "        best_loss = loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "trans_train = transforms.Compose([transforms.ToPILImage(),\n",
    "                                  transforms.Pad(64, padding_mode='reflect'),\n",
    "                                  transforms.RandomHorizontalFlip(), \n",
    "                                  transforms.RandomVerticalFlip(),\n",
    "                                  transforms.RandomRotation(20), \n",
    "                                  transforms.Resize(224, interpolation = 2),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n",
    "\n",
    "trans_valid = transforms.Compose([transforms.ToPILImage(),                    \n",
    "                                  transforms.Pad(64, padding_mode='reflect'),\n",
    "                                  transforms.Resize(224, interpolation = 2),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n",
    "\n",
    "dataset_train = MyDataset(df_data=train, transform=trans_train)\n",
    "dataset_valid = MyDataset(df_data=val,transform=trans_valid)\n",
    "\n",
    "loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we create the model and train it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torchvision import models\n",
    "\n",
    "def convrelu(in_channels, out_channels, kernel, padding):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(in_channels, out_channels, kernel, padding=padding),\n",
    "        nn.ReLU(inplace=True),\n",
    "    )\n",
    "\n",
    "\n",
    "class ResNetUNet(nn.Module):\n",
    "    def __init__(self, n_class):\n",
    "        super().__init__()\n",
    "\n",
    "        self.base_model = models.resnet18(pretrained=True)\n",
    "        self.base_layers = list(self.base_model.children())\n",
    "\n",
    "        self.layer0 = nn.Sequential(*self.base_layers[:3]) # size=(N, 64, x.H/2, x.W/2)\n",
    "        self.layer0_1x1 = convrelu(64, 64, 1, 0)\n",
    "        self.layer1 = nn.Sequential(*self.base_layers[3:5]) # size=(N, 64, x.H/4, x.W/4)\n",
    "        self.layer1_1x1 = convrelu(64, 64, 1, 0)\n",
    "        self.layer2 = self.base_layers[5]  # size=(N, 128, x.H/8, x.W/8)\n",
    "        self.layer2_1x1 = convrelu(128, 128, 1, 0)\n",
    "        self.layer3 = self.base_layers[6]  # size=(N, 256, x.H/16, x.W/16)\n",
    "        self.layer3_1x1 = convrelu(256, 256, 1, 0)\n",
    "        self.layer4 = self.base_layers[7]  # size=(N, 512, x.H/32, x.W/32)\n",
    "        self.layer4_1x1 = convrelu(512, 512, 1, 0)\n",
    "\n",
    "        self.upsample = nn.Upsample(scale_factor=2, mode='bicubic', align_corners=True) # this is now not being used. Keeping it for testing only.\n",
    "        \n",
    "        # o = s*(n-1) + f - 2*p\n",
    "        self.upsample_4_3 = nn.ConvTranspose2d(in_channels = 512, out_channels = 512, kernel_size = 4, stride = 2, padding = 1)  # 7 -> 14\n",
    "        self.upsample_3_2 = nn.ConvTranspose2d(in_channels = 512, out_channels = 512, kernel_size = 4, stride = 2, padding = 1) # 14 -> 28\n",
    "        self.upsample_2_1 = nn.ConvTranspose2d(in_channels = 256, out_channels = 256, kernel_size = 4, stride = 2, padding = 1) # 28 -> 56\n",
    "        self.upsample_1_0 = nn.ConvTranspose2d(in_channels = 256, out_channels = 256, kernel_size = 4, stride = 2, padding = 1) # 56 -> 112\n",
    "        self.upsample_0_f = nn.ConvTranspose2d(in_channels = 128, out_channels = 128, kernel_size = 4, stride = 2, padding = 1) # 112 -> 224\n",
    "        \n",
    "        self.conv_up3 = convrelu(256 + 512, 512, 3, 1)\n",
    "        self.conv_up2 = convrelu(128 + 512, 256, 3, 1)\n",
    "        self.conv_up1 = convrelu(64 + 256, 256, 3, 1)\n",
    "        self.conv_up0 = convrelu(64 + 256, 128, 3, 1)\n",
    "\n",
    "        self.conv_original_size0 = convrelu(3, 64, 3, 1)\n",
    "        self.conv_original_size1 = convrelu(64, 64, 3, 1)\n",
    "        self.conv_original_size2 = convrelu(64 + 128, 64, 3, 1)\n",
    "\n",
    "        self.conv_last = nn.Conv2d(64, n_class, 1)\n",
    "        \n",
    "        self.avg = nn.AvgPool2d(8) # 8x2x28x28\n",
    "        self.fc = nn.Linear(n_class*28*28,2)\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "\n",
    "    def forward(self, input):\n",
    "        x_original = self.conv_original_size0(input)\n",
    "        x_original = self.conv_original_size1(x_original)\n",
    "\n",
    "        layer0 = self.layer0(input)\n",
    "        layer1 = self.layer1(layer0)\n",
    "        layer2 = self.layer2(layer1)\n",
    "        layer3 = self.layer3(layer2)\n",
    "        layer4 = self.layer4(layer3)\n",
    "\n",
    "        layer4 = self.layer4_1x1(layer4)\n",
    "        x = self.upsample(layer4)\n",
    "        \n",
    "        layer3 = self.layer3_1x1(layer3)\n",
    "        x = torch.cat([x, layer3], dim=1)\n",
    "        x = self.conv_up3(x)\n",
    "        x = self.upsample(x)\n",
    "        layer2 = self.layer2_1x1(layer2)\n",
    "        x = torch.cat([x, layer2], dim=1)\n",
    "        x = self.conv_up2(x)\n",
    "        x = self.upsample(x)\n",
    "        \n",
    "        layer1 = self.layer1_1x1(layer1)\n",
    "        x = torch.cat([x, layer1], dim=1)\n",
    "        x = self.conv_up1(x)\n",
    "        x = self.upsample(x)  #       \n",
    "        layer0 = self.layer0_1x1(layer0)\n",
    "        x = torch.cat([x, layer0], dim=1)\n",
    "        x = self.conv_up0(x)\n",
    "        x = self.upsample(x)\n",
    "        x = torch.cat([x, x_original], dim=1)\n",
    "        x = self.conv_original_size2(x)\n",
    "\n",
    "        out = self.conv_last(x)  \n",
    "        out = self.avg(out)\n",
    "        out = out.view(-1,num_classes*28*28)\n",
    "        out = self.fc(out)\n",
    "        # out = self.softmax(out)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /tmp/.cache/torch/checkpoints/resnet18-5c106cde.pth\n",
      "100%|██████████| 44.7M/44.7M [00:00<00:00, 95.9MB/s]\n"
     ]
    }
   ],
   "source": [
    "model = ResNetUNet(num_classes).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adamax(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading pre-trained model state...\n",
      "Done!\n",
      "Downloaded model state loaded successfully.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "url = \"https://onedrive.live.com/download?cid=E5569BBAB912B6A5&resid=E5569BBAB912B6A5%21484258&authkey=AAyx18QtkAxB4D8\"\n",
    "file_name = \"download.dct\"\n",
    "print(\"Downloading pre-trained model state...\")\n",
    "urllib.request.urlretrieve(url, file_name)\n",
    "print(\"Done!\")\n",
    "model.load_state_dict(torch.load(file_name))\n",
    "import os.path\n",
    "if os.path.exists(file_name):\n",
    "    try:\n",
    "        model.load_state_dict(torch.load(file_name))\n",
    "        print(\"Downloaded model state loaded successfully.\")\n",
    "    except:\n",
    "        print(\"Error loading state dict.\")\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point, we download a pre-trained model that was trained during the previous sesstions. OneDrive allows direct downloads. Simply open the file in the browser in OneDrive and copy the link in the **embed** code. In the URL string, replace \"embed\" with \"download\" and you are good to go. Every time you upload a file, simply change the URL.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1  started...\n",
      "Epoch [1/50], Step [100/396], Loss: 0.0171\n",
      "=> Loss reduced by:\t 8.982893377542496\n",
      "   Saving model state\n",
      "Epoch [1/50], Step [200/396], Loss: 0.0009\n",
      "=> Loss reduced by:\t 0.01619914174079895\n",
      "   Saving model state\n",
      "Epoch [1/50], Step [300/396], Loss: 0.0005\n",
      "=> Loss reduced by:\t 0.00038039684295654297\n",
      "   Saving model state\n",
      "Epoch done in  0:04:49.755447\n",
      "Epoch  2  started...\n",
      "Epoch [2/50], Step [100/396], Loss: 0.0009\n",
      "Epoch [2/50], Step [200/396], Loss: 0.0006\n",
      "Epoch [2/50], Step [300/396], Loss: 0.0002\n",
      "=> Loss reduced by:\t 0.000339701771736145\n",
      "   Saving model state\n",
      "Epoch done in  0:04:09.579911\n",
      "Epoch  3  started...\n",
      "Epoch [3/50], Step [100/396], Loss: 0.0106\n",
      "Epoch [3/50], Step [200/396], Loss: 0.0009\n",
      "Epoch [3/50], Step [300/396], Loss: 0.0005\n",
      "Epoch done in  0:04:07.454669\n",
      "Epoch  4  started...\n",
      "Epoch [4/50], Step [100/396], Loss: 0.0000\n",
      "=> Loss reduced by:\t 0.00018502771854400635\n",
      "   Saving model state\n",
      "Epoch [4/50], Step [200/396], Loss: 0.0004\n",
      "Epoch [4/50], Step [300/396], Loss: 0.0007\n",
      "Epoch done in  0:04:07.416133\n",
      "Epoch  5  started...\n",
      "Epoch [5/50], Step [100/396], Loss: 0.0117\n",
      "Epoch [5/50], Step [200/396], Loss: 0.0047\n",
      "Epoch [5/50], Step [300/396], Loss: 0.0015\n",
      "Epoch done in  0:04:06.627470\n",
      "Epoch  6  started...\n",
      "Epoch [6/50], Step [100/396], Loss: 0.0008\n",
      "Epoch [6/50], Step [200/396], Loss: 0.0021\n",
      "Epoch [6/50], Step [300/396], Loss: 0.0113\n",
      "Epoch done in  0:04:06.212163\n",
      "Epoch  7  started...\n",
      "Epoch [7/50], Step [100/396], Loss: 0.0017\n",
      "Epoch [7/50], Step [200/396], Loss: 0.0005\n",
      "Epoch [7/50], Step [300/396], Loss: 0.0069\n",
      "Epoch done in  0:04:06.144202\n",
      "Epoch  8  started...\n",
      "Epoch [8/50], Step [100/396], Loss: 0.0069\n",
      "Epoch [8/50], Step [200/396], Loss: 0.0036\n",
      "Epoch [8/50], Step [300/396], Loss: 0.0001\n",
      "Epoch done in  0:04:05.843597\n",
      "Epoch  9  started...\n",
      "Epoch [9/50], Step [100/396], Loss: 0.0024\n",
      "Epoch [9/50], Step [200/396], Loss: 0.0035\n",
      "Epoch [9/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:06.139758\n",
      "Epoch  10  started...\n",
      "Epoch [10/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [10/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [10/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:08.455894\n",
      "Epoch  11  started...\n",
      "Epoch [11/50], Step [100/396], Loss: 0.0006\n",
      "Epoch [11/50], Step [200/396], Loss: 0.0092\n",
      "Epoch [11/50], Step [300/396], Loss: 0.0005\n",
      "Epoch done in  0:04:08.774032\n",
      "Epoch  12  started...\n",
      "Epoch [12/50], Step [100/396], Loss: 0.0011\n",
      "Epoch [12/50], Step [200/396], Loss: 0.0002\n",
      "Epoch [12/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:07.306192\n",
      "Epoch  13  started...\n",
      "Epoch [13/50], Step [100/396], Loss: 0.0062\n",
      "Epoch [13/50], Step [200/396], Loss: 0.0003\n",
      "Epoch [13/50], Step [300/396], Loss: 0.0007\n",
      "Epoch done in  0:04:07.161977\n",
      "Epoch  14  started...\n",
      "Epoch [14/50], Step [100/396], Loss: 0.0034\n",
      "Epoch [14/50], Step [200/396], Loss: 0.0002\n",
      "Epoch [14/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:08.429324\n",
      "Epoch  15  started...\n",
      "Epoch [15/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [15/50], Step [200/396], Loss: 0.0003\n",
      "Epoch [15/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:10.985280\n",
      "Epoch  16  started...\n",
      "Epoch [16/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [16/50], Step [200/396], Loss: 0.0005\n",
      "Epoch [16/50], Step [300/396], Loss: 0.0026\n",
      "Epoch done in  0:04:11.216455\n",
      "Epoch  17  started...\n",
      "Epoch [17/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [17/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [17/50], Step [300/396], Loss: 0.0016\n",
      "Epoch done in  0:04:10.032719\n",
      "Epoch  18  started...\n",
      "Epoch [18/50], Step [100/396], Loss: 0.0216\n",
      "Epoch [18/50], Step [200/396], Loss: 0.0148\n",
      "Epoch [18/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:08.043921\n",
      "Epoch  19  started...\n",
      "Epoch [19/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [19/50], Step [200/396], Loss: 0.0019\n",
      "Epoch [19/50], Step [300/396], Loss: 0.0013\n",
      "Epoch done in  0:04:07.093691\n",
      "Epoch  20  started...\n",
      "Epoch [20/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [20/50], Step [200/396], Loss: 0.0016\n",
      "Epoch [20/50], Step [300/396], Loss: 0.0205\n",
      "Epoch done in  0:04:05.691306\n",
      "Epoch  21  started...\n",
      "Epoch [21/50], Step [100/396], Loss: 0.0005\n",
      "Epoch [21/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [21/50], Step [300/396], Loss: 0.0096\n",
      "Epoch done in  0:04:05.517240\n",
      "Epoch  22  started...\n",
      "Epoch [22/50], Step [100/396], Loss: 0.0004\n",
      "Epoch [22/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [22/50], Step [300/396], Loss: 0.0068\n",
      "Epoch done in  0:04:05.716001\n",
      "Epoch  23  started...\n",
      "Epoch [23/50], Step [100/396], Loss: 0.0009\n",
      "Epoch [23/50], Step [200/396], Loss: 0.0008\n",
      "Epoch [23/50], Step [300/396], Loss: 0.0007\n",
      "Epoch done in  0:04:05.908231\n",
      "Epoch  24  started...\n",
      "Epoch [24/50], Step [100/396], Loss: 0.0008\n",
      "Epoch [24/50], Step [200/396], Loss: 0.0003\n",
      "Epoch [24/50], Step [300/396], Loss: 0.0007\n",
      "Epoch done in  0:04:06.095047\n",
      "Epoch  25  started...\n",
      "Epoch [25/50], Step [100/396], Loss: 0.0008\n",
      "Epoch [25/50], Step [200/396], Loss: 0.0052\n",
      "Epoch [25/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:07.401150\n",
      "Epoch  26  started...\n",
      "Epoch [26/50], Step [100/396], Loss: 0.0003\n",
      "Epoch [26/50], Step [200/396], Loss: 0.0028\n",
      "Epoch [26/50], Step [300/396], Loss: 0.0355\n",
      "Epoch done in  0:04:08.829151\n",
      "Epoch  27  started...\n",
      "Epoch [27/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [27/50], Step [200/396], Loss: 0.0011\n",
      "Epoch [27/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:06.922159\n",
      "Epoch  28  started...\n",
      "Epoch [28/50], Step [100/396], Loss: 0.0014\n",
      "Epoch [28/50], Step [200/396], Loss: 0.0004\n",
      "Epoch [28/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:06.756142\n",
      "Epoch  29  started...\n",
      "Epoch [29/50], Step [100/396], Loss: 0.1887\n",
      "Epoch [29/50], Step [200/396], Loss: 0.0114\n",
      "Epoch [29/50], Step [300/396], Loss: 0.0010\n",
      "Epoch done in  0:04:07.410521\n",
      "Epoch  30  started...\n",
      "Epoch [30/50], Step [100/396], Loss: 0.0008\n",
      "Epoch [30/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [30/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:10.210785\n",
      "Epoch  31  started...\n",
      "Epoch [31/50], Step [100/396], Loss: 0.0008\n",
      "Epoch [31/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [31/50], Step [300/396], Loss: 0.0013\n",
      "Epoch done in  0:04:10.673921\n",
      "Epoch  32  started...\n",
      "Epoch [32/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [32/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [32/50], Step [300/396], Loss: 0.0001\n",
      "Epoch done in  0:04:09.229909\n",
      "Epoch  33  started...\n",
      "Epoch [33/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [33/50], Step [200/396], Loss: 0.0004\n",
      "Epoch [33/50], Step [300/396], Loss: 0.0009\n",
      "Epoch done in  0:04:07.536515\n",
      "Epoch  34  started...\n",
      "Epoch [34/50], Step [100/396], Loss: 0.0007\n",
      "Epoch [34/50], Step [200/396], Loss: 0.0006\n",
      "Epoch [34/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:06.237371\n",
      "Epoch  35  started...\n",
      "Epoch [35/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [35/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [35/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:05.733296\n",
      "Epoch  36  started...\n",
      "Epoch [36/50], Step [100/396], Loss: 0.0011\n",
      "Epoch [36/50], Step [200/396], Loss: 0.0030\n",
      "Epoch [36/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:05.395364\n",
      "Epoch  37  started...\n",
      "Epoch [37/50], Step [100/396], Loss: 0.1637\n",
      "Epoch [37/50], Step [200/396], Loss: 0.0009\n",
      "Epoch [37/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:05.542555\n",
      "Epoch  38  started...\n",
      "Epoch [38/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [38/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [38/50], Step [300/396], Loss: 0.0006\n",
      "Epoch done in  0:04:05.197887\n",
      "Epoch  39  started...\n",
      "Epoch [39/50], Step [100/396], Loss: 0.0002\n",
      "Epoch [39/50], Step [200/396], Loss: 0.0003\n",
      "Epoch [39/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:05.705898\n",
      "Epoch  40  started...\n",
      "Epoch [40/50], Step [100/396], Loss: 0.0047\n",
      "Epoch [40/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [40/50], Step [300/396], Loss: 0.0001\n",
      "Epoch done in  0:04:07.367506\n",
      "Epoch  41  started...\n",
      "Epoch [41/50], Step [100/396], Loss: 0.0004\n",
      "Epoch [41/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [41/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:08.268543\n",
      "Epoch  42  started...\n",
      "Epoch [42/50], Step [100/396], Loss: 0.0017\n",
      "Epoch [42/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [42/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:07.301824\n",
      "Epoch  43  started...\n",
      "Epoch [43/50], Step [100/396], Loss: 0.0004\n",
      "Epoch [43/50], Step [200/396], Loss: 0.0002\n",
      "Epoch [43/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:07.234006\n",
      "Epoch  44  started...\n",
      "Epoch [44/50], Step [100/396], Loss: 0.0001\n",
      "Epoch [44/50], Step [200/396], Loss: 0.0000\n",
      "Epoch [44/50], Step [300/396], Loss: 0.0040\n",
      "Epoch done in  0:04:06.834894\n",
      "Epoch  45  started...\n",
      "Epoch [45/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [45/50], Step [200/396], Loss: 0.0002\n",
      "Epoch [45/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:09.293087\n",
      "Epoch  46  started...\n",
      "Epoch [46/50], Step [100/396], Loss: 0.0025\n",
      "Epoch [46/50], Step [200/396], Loss: 0.0003\n",
      "Epoch [46/50], Step [300/396], Loss: 0.0000\n",
      "Epoch done in  0:04:09.366515\n",
      "Epoch  47  started...\n",
      "Epoch [47/50], Step [100/396], Loss: 0.0289\n",
      "Epoch [47/50], Step [200/396], Loss: 0.0001\n",
      "Epoch [47/50], Step [300/396], Loss: 0.0012\n",
      "Epoch done in  0:04:09.380747\n",
      "Epoch  48  started...\n",
      "Epoch [48/50], Step [100/396], Loss: 0.0006\n",
      "Epoch [48/50], Step [200/396], Loss: 0.1089\n",
      "Epoch [48/50], Step [300/396], Loss: 0.0019\n",
      "Epoch done in  0:04:07.028648\n",
      "Epoch  49  started...\n",
      "Epoch [49/50], Step [100/396], Loss: 0.0000\n",
      "Epoch [49/50], Step [200/396], Loss: 0.0006\n",
      "Epoch [49/50], Step [300/396], Loss: 0.0028\n",
      "Epoch done in  0:04:06.434529\n",
      "Epoch  50  started...\n",
      "Epoch [50/50], Step [100/396], Loss: 0.0004\n",
      "Epoch [50/50], Step [200/396], Loss: 0.0004\n",
      "Epoch [50/50], Step [300/396], Loss: 0.0002\n",
      "Epoch done in  0:04:06.335102\n",
      "Training done in  3:26:55.245887\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "timestamp_train_start = time.time()\n",
    "\n",
    "\n",
    "loss_hist = []\n",
    "best_loss = 9\n",
    "total_step = len(loader_train)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    timestamp_epoch_start = time.time()\n",
    "    print(\"Epoch \", epoch+1,\" started...\")\n",
    "    for i, (images, labels) in enumerate(loader_train):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "                \n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "       \n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))    \n",
    "             # Checkpointing\n",
    "            loss_hist.append(float(loss.item())) #add current loss value.\n",
    "            save_checkpoint(model.state_dict(),float(loss.item()))\n",
    "                    \n",
    "    timestamp_epoch_end = time.time()\n",
    "    print(\"Epoch done in \",str(datetime.timedelta(seconds=(timestamp_epoch_end - timestamp_epoch_start))))\n",
    "    \n",
    "timestamp_train_end = time.time()\n",
    "print(\"Training done in \",str(datetime.timedelta(seconds=(timestamp_train_end - timestamp_train_start))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the test images: 99.49431099873578 %\n"
     ]
    }
   ],
   "source": [
    "#load the best model\n",
    "model.load_state_dict(torch.load(\"state_dict.dct\"))\n",
    "\n",
    "model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in loader_valid:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "          \n",
    "    print('Test Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n",
    "\n",
    "# Save the model checkpoint\n",
    "torch.save(model.state_dict(), 'final_state.dct')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total execution time:  3:27:53.668974\n"
     ]
    }
   ],
   "source": [
    "timestamp_exec_end = time.time()\n",
    "print(\"Total execution time: \",str(datetime.timedelta(seconds=(timestamp_exec_end - timestamp_exec_start))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}