breakhisv1-xception.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed  7909 files\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "labels\n",
       "0     444\n",
       "1    1014\n",
       "2     569\n",
       "3     453\n",
       "4    3451\n",
       "5     626\n",
       "6     792\n",
       "7     560\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Xception\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        pass\n",
    "        # print(os.path.join(dirname, filename))\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "timestamp_exec_start = time.time()\n",
    "\n",
    "\n",
    "files = []\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        if (filename[-3:] == 'png'):\n",
    "            files.append(os.path.join(dirname, filename))\n",
    "print(\"Processed \",len(files),\"files\")\n",
    "\n",
    "\n",
    "labels_dict = {\"B_A-\":0,\"B_F-\":1,\"B_TA\":2,\"B_PT\":3,\"M_DC\":4,\"M_LC\":5,\"M_MC\":6,\"M_PC\":7}  # for 8 class problem\n",
    "labels_dict_simple = {\"B\":0,\"M\":1}                                                       # for 2 class problem\n",
    "REDUCED_CLASSES = False\n",
    "\n",
    "X = []\n",
    "Y = []\n",
    "for f in files:\n",
    "    x = f.split(\"/\") # break up the path\n",
    "    x = x[-1:][0]    # extract the file name\n",
    "    X.append(str(f))\n",
    "    if REDUCED_CLASSES:\n",
    "        Y.append(int(labels_dict_simple[x[4]]))\n",
    "    else:\n",
    "        Y.append(int(labels_dict[x[4:8]]))\n",
    "\n",
    "data = {\"images\":X,\"labels\":Y}\n",
    "images_df = pd.DataFrame(data, columns = ['images','labels'])\n",
    "images_df.groupby(\"labels\")[\"labels\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6327, 1582)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train, val = train_test_split(images_df, stratify=images_df.labels, test_size=0.2)\n",
    "len(train), len(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, df_data,transform=None):\n",
    "        super().__init__()\n",
    "        self.df = df_data.values\n",
    "        \n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        img_path,label = self.df[index]\n",
    "        \n",
    "        image = cv2.imread(img_path)\n",
    "        image = cv2.resize(image, (224,224))\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Parameters for model\n",
    "\n",
    "# Hyper parameters\n",
    "num_epochs = 150\n",
    "num_classes = 8\n",
    "batch_size = 16\n",
    "learning_rate = 0.0002\n",
    "\n",
    "# Device configuration\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utility function for saving model\n",
    "# During training, the loss values are stored in a list.\n",
    "# We check the last two values to see if the loss has reduced.\n",
    "def save_checkpoint(state, loss):\n",
    "    global best_loss\n",
    "    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n",
    "    if best_loss>=loss:        \n",
    "        print (\"=> Loss reduced by:\\t\",best_loss - loss)\n",
    "        print(\"   Saving model state\")\n",
    "        torch.save(state, \"state_dict.dct\")  # save checkpoint\n",
    "        best_loss = loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "trans_train = transforms.Compose([transforms.ToPILImage(),\n",
    "                                  transforms.Pad(64, padding_mode='reflect'),\n",
    "                                  transforms.RandomHorizontalFlip(), \n",
    "                                  transforms.RandomVerticalFlip(),\n",
    "                                  transforms.RandomRotation(20), \n",
    "                                  transforms.Resize(224, interpolation = 2),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n",
    "\n",
    "trans_valid = transforms.Compose([transforms.ToPILImage(),                    \n",
    "                                  transforms.Pad(64, padding_mode='reflect'),\n",
    "                                  transforms.Resize(224, interpolation = 2),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n",
    "\n",
    "dataset_train = MyDataset(df_data=train, transform=trans_train)\n",
    "dataset_valid = MyDataset(df_data=val,transform=trans_valid)\n",
    "\n",
    "loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we create the model and train it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.utils import save_image\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import os\n",
    "import glob\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils import data as D\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import random\n",
    "#%%\n",
    "class depthwise_separable_conv(nn.Module):\n",
    "    def __init__(self, nin, nout, kernel_size, padding, bias=False):\n",
    "        super(depthwise_separable_conv, self).__init__()\n",
    "        self.depthwise = nn.Conv2d(nin, nin, kernel_size=kernel_size, padding=padding, groups=nin, bias=bias)\n",
    "        self.pointwise = nn.Conv2d(nin, nout, kernel_size=1, bias=bias)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.depthwise(x)\n",
    "        out = self.pointwise(out)\n",
    "        return out\n",
    "# %%\n",
    "class Xception(nn.Module):\n",
    "    def __init__(self, input_channel, num_classes=8):\n",
    "        super(Xception, self).__init__()\n",
    "        \n",
    "        # Entry Flow\n",
    "        self.entry_flow_1 = nn.Sequential(\n",
    "            nn.Conv2d(input_channel, 32, kernel_size=3, stride=2, padding=1, bias=False),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(True),\n",
    "            \n",
    "            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(True)\n",
    "        )\n",
    "        \n",
    "        self.entry_flow_2 = nn.Sequential(\n",
    "            depthwise_separable_conv(64, 128, 3, 1),\n",
    "            nn.BatchNorm2d(128),\n",
    "            nn.ReLU(True),\n",
    "            \n",
    "            depthwise_separable_conv(128, 128, 3, 1),\n",
    "            nn.BatchNorm2d(128),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        )\n",
    "        \n",
    "        self.entry_flow_2_residual = nn.Conv2d(64, 128, kernel_size=1, stride=2, padding=0)\n",
    "        \n",
    "        self.entry_flow_3 = nn.Sequential(\n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(128, 256, 3, 1),\n",
    "            nn.BatchNorm2d(256),\n",
    "            \n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(256, 256, 3, 1),\n",
    "            nn.BatchNorm2d(256),\n",
    "            \n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        )\n",
    "        \n",
    "        self.entry_flow_3_residual = nn.Conv2d(128, 256, kernel_size=1, stride=2, padding=0)\n",
    "        \n",
    "        self.entry_flow_4 = nn.Sequential(\n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(256, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728),\n",
    "            \n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728),\n",
    "            \n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        )\n",
    "        \n",
    "        self.entry_flow_4_residual = nn.Conv2d(256, 728, kernel_size=1, stride=2, padding=0)\n",
    "        \n",
    "        # Middle Flow\n",
    "        self.middle_flow = nn.Sequential(\n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728),\n",
    "            \n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728),\n",
    "            \n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728)\n",
    "        )\n",
    "        \n",
    "        # Exit Flow\n",
    "        self.exit_flow_1 = nn.Sequential(\n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 728, 3, 1),\n",
    "            nn.BatchNorm2d(728),\n",
    "            \n",
    "            nn.ReLU(True),\n",
    "            depthwise_separable_conv(728, 1024, 3, 1),\n",
    "            nn.BatchNorm2d(1024),\n",
    "            \n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        )\n",
    "        self.exit_flow_1_residual = nn.Conv2d(728, 1024, kernel_size=1, stride=2, padding=0)\n",
    "        self.exit_flow_2 = nn.Sequential(\n",
    "            depthwise_separable_conv(1024, 1536, 3, 1),\n",
    "            nn.BatchNorm2d(1536),\n",
    "            nn.ReLU(True),\n",
    "            \n",
    "            depthwise_separable_conv(1536, 2048, 3, 1),\n",
    "            nn.BatchNorm2d(2048),\n",
    "            nn.ReLU(True)\n",
    "        )\n",
    "        \n",
    "        self.linear = nn.Linear(2048, num_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        entry_out1 = self.entry_flow_1(x)\n",
    "        entry_out2 = self.entry_flow_2(entry_out1) + self.entry_flow_2_residual(entry_out1)\n",
    "        entry_out3 = self.entry_flow_3(entry_out2) + self.entry_flow_3_residual(entry_out2)\n",
    "        entry_out = self.entry_flow_4(entry_out3) + self.entry_flow_4_residual(entry_out3)\n",
    "        \n",
    "        middle_out = self.middle_flow(entry_out) + entry_out\n",
    "        \n",
    "        for i in range(7):\n",
    "          middle_out = self.middle_flow(middle_out) + middle_out\n",
    "\n",
    "        exit_out1 = self.exit_flow_1(middle_out) + self.exit_flow_1_residual(middle_out)\n",
    "        exit_out2 = self.exit_flow_2(exit_out1)\n",
    "\n",
    "        exit_avg_pool = F.adaptive_avg_pool2d(exit_out2, (1, 1))                \n",
    "        exit_avg_pool_flat = exit_avg_pool.view(exit_avg_pool.size(0), -1)\n",
    "\n",
    "        output = self.linear(exit_avg_pool_flat)\n",
    "        \n",
    "        return output\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 8])\n"
     ]
    }
   ],
   "source": [
    "model = Xception(3, 8).to(device)\n",
    "\n",
    "dummy = torch.ones((16,3,224,224)).cuda()\n",
    "out = model(dummy)\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adamax(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1  started...\n",
      "Epoch [1/150], Step [100/396], Loss: 1.1357\n",
      "=> Loss reduced by:\t 7.8643105030059814\n",
      "   Saving model state\n",
      "Epoch [1/150], Step [200/396], Loss: 1.7973\n",
      "Epoch [1/150], Step [300/396], Loss: 1.0914\n",
      "=> Loss reduced by:\t 0.04430079460144043\n",
      "   Saving model state\n",
      "Epoch done in  0:02:25.752647\n",
      "Epoch  2  started...\n",
      "Epoch [2/150], Step [100/396], Loss: 1.1945\n",
      "Epoch [2/150], Step [200/396], Loss: 1.1725\n",
      "Epoch [2/150], Step [300/396], Loss: 1.5153\n",
      "Epoch done in  0:01:55.069613\n",
      "Epoch  3  started...\n",
      "Epoch [3/150], Step [100/396], Loss: 0.8449\n",
      "=> Loss reduced by:\t 0.24648046493530273\n",
      "   Saving model state\n",
      "Epoch [3/150], Step [200/396], Loss: 0.6813\n",
      "=> Loss reduced by:\t 0.16365259885787964\n",
      "   Saving model state\n",
      "Epoch [3/150], Step [300/396], Loss: 0.9344\n",
      "Epoch done in  0:01:54.423730\n",
      "Epoch  4  started...\n",
      "Epoch [4/150], Step [100/396], Loss: 0.9273\n",
      "Epoch [4/150], Step [200/396], Loss: 0.9198\n",
      "Epoch [4/150], Step [300/396], Loss: 1.1148\n",
      "Epoch done in  0:01:54.355351\n",
      "Epoch  5  started...\n",
      "Epoch [5/150], Step [100/396], Loss: 1.1623\n",
      "Epoch [5/150], Step [200/396], Loss: 0.8589\n",
      "Epoch [5/150], Step [300/396], Loss: 0.7368\n",
      "Epoch done in  0:01:53.012135\n",
      "Epoch  6  started...\n",
      "Epoch [6/150], Step [100/396], Loss: 0.9501\n",
      "Epoch [6/150], Step [200/396], Loss: 0.9612\n",
      "Epoch [6/150], Step [300/396], Loss: 0.9488\n",
      "Epoch done in  0:01:53.260871\n",
      "Epoch  7  started...\n",
      "Epoch [7/150], Step [100/396], Loss: 0.5412\n",
      "=> Loss reduced by:\t 0.14008110761642456\n",
      "   Saving model state\n",
      "Epoch [7/150], Step [200/396], Loss: 1.2643\n",
      "Epoch [7/150], Step [300/396], Loss: 0.5449\n",
      "Epoch done in  0:01:52.312479\n",
      "Epoch  8  started...\n",
      "Epoch [8/150], Step [100/396], Loss: 0.9773\n",
      "Epoch [8/150], Step [200/396], Loss: 0.6218\n",
      "Epoch [8/150], Step [300/396], Loss: 0.8695\n",
      "Epoch done in  0:01:52.871434\n",
      "Epoch  9  started...\n",
      "Epoch [9/150], Step [100/396], Loss: 0.6562\n",
      "Epoch [9/150], Step [200/396], Loss: 0.5080\n",
      "=> Loss reduced by:\t 0.033217430114746094\n",
      "   Saving model state\n",
      "Epoch [9/150], Step [300/396], Loss: 0.9002\n",
      "Epoch done in  0:01:53.427155\n",
      "Epoch  10  started...\n",
      "Epoch [10/150], Step [100/396], Loss: 0.8815\n",
      "Epoch [10/150], Step [200/396], Loss: 0.5659\n",
      "Epoch [10/150], Step [300/396], Loss: 0.6730\n",
      "Epoch done in  0:01:53.680653\n",
      "Epoch  11  started...\n",
      "Epoch [11/150], Step [100/396], Loss: 0.8045\n",
      "Epoch [11/150], Step [200/396], Loss: 0.4990\n",
      "=> Loss reduced by:\t 0.008986860513687134\n",
      "   Saving model state\n",
      "Epoch [11/150], Step [300/396], Loss: 1.0383\n",
      "Epoch done in  0:01:54.004354\n",
      "Epoch  12  started...\n",
      "Epoch [12/150], Step [100/396], Loss: 0.7884\n",
      "Epoch [12/150], Step [200/396], Loss: 0.5076\n",
      "Epoch [12/150], Step [300/396], Loss: 0.5756\n",
      "Epoch done in  0:01:52.818224\n",
      "Epoch  13  started...\n",
      "Epoch [13/150], Step [100/396], Loss: 0.5282\n",
      "Epoch [13/150], Step [200/396], Loss: 0.9928\n",
      "Epoch [13/150], Step [300/396], Loss: 0.4725\n",
      "=> Loss reduced by:\t 0.026482760906219482\n",
      "   Saving model state\n",
      "Epoch done in  0:01:53.330597\n",
      "Epoch  14  started...\n",
      "Epoch [14/150], Step [100/396], Loss: 0.3439\n",
      "=> Loss reduced by:\t 0.12860450148582458\n",
      "   Saving model state\n",
      "Epoch [14/150], Step [200/396], Loss: 0.7893\n",
      "Epoch [14/150], Step [300/396], Loss: 0.6596\n",
      "Epoch done in  0:01:53.414559\n",
      "Epoch  15  started...\n",
      "Epoch [15/150], Step [100/396], Loss: 0.5581\n",
      "Epoch [15/150], Step [200/396], Loss: 0.2294\n",
      "=> Loss reduced by:\t 0.11446365714073181\n",
      "   Saving model state\n",
      "Epoch [15/150], Step [300/396], Loss: 0.6653\n",
      "Epoch done in  0:01:53.584066\n",
      "Epoch  16  started...\n",
      "Epoch [16/150], Step [100/396], Loss: 0.6539\n",
      "Epoch [16/150], Step [200/396], Loss: 0.3836\n",
      "Epoch [16/150], Step [300/396], Loss: 0.3423\n",
      "Epoch done in  0:01:53.624346\n",
      "Epoch  17  started...\n",
      "Epoch [17/150], Step [100/396], Loss: 0.2882\n",
      "Epoch [17/150], Step [200/396], Loss: 0.4497\n",
      "Epoch [17/150], Step [300/396], Loss: 0.5857\n",
      "Epoch done in  0:01:53.075406\n",
      "Epoch  18  started...\n",
      "Epoch [18/150], Step [100/396], Loss: 0.4099\n",
      "Epoch [18/150], Step [200/396], Loss: 0.3949\n",
      "Epoch [18/150], Step [300/396], Loss: 0.5454\n",
      "Epoch done in  0:01:52.952239\n",
      "Epoch  19  started...\n",
      "Epoch [19/150], Step [100/396], Loss: 0.6448\n",
      "Epoch [19/150], Step [200/396], Loss: 0.2400\n",
      "Epoch [19/150], Step [300/396], Loss: 0.3139\n",
      "Epoch done in  0:01:52.969032\n",
      "Epoch  20  started...\n",
      "Epoch [20/150], Step [100/396], Loss: 0.2768\n",
      "Epoch [20/150], Step [200/396], Loss: 0.2542\n",
      "Epoch [20/150], Step [300/396], Loss: 0.3828\n",
      "Epoch done in  0:01:52.959114\n",
      "Epoch  21  started...\n",
      "Epoch [21/150], Step [100/396], Loss: 0.3249\n",
      "Epoch [21/150], Step [200/396], Loss: 0.4901\n",
      "Epoch [21/150], Step [300/396], Loss: 0.2799\n",
      "Epoch done in  0:01:52.585794\n",
      "Epoch  22  started...\n",
      "Epoch [22/150], Step [100/396], Loss: 0.4333\n",
      "Epoch [22/150], Step [200/396], Loss: 0.0507\n",
      "=> Loss reduced by:\t 0.1786801815032959\n",
      "   Saving model state\n",
      "Epoch [22/150], Step [300/396], Loss: 0.4234\n",
      "Epoch done in  0:01:52.268409\n",
      "Epoch  23  started...\n",
      "Epoch [23/150], Step [100/396], Loss: 0.2433\n",
      "Epoch [23/150], Step [200/396], Loss: 0.2519\n",
      "Epoch [23/150], Step [300/396], Loss: 0.1779\n",
      "Epoch done in  0:01:52.358300\n",
      "Epoch  24  started...\n",
      "Epoch [24/150], Step [100/396], Loss: 0.4280\n",
      "Epoch [24/150], Step [200/396], Loss: 0.1904\n",
      "Epoch [24/150], Step [300/396], Loss: 0.2986\n",
      "Epoch done in  0:01:52.889874\n",
      "Epoch  25  started...\n",
      "Epoch [25/150], Step [100/396], Loss: 0.3336\n",
      "Epoch [25/150], Step [200/396], Loss: 0.2190\n",
      "Epoch [25/150], Step [300/396], Loss: 0.2924\n",
      "Epoch done in  0:01:53.436230\n",
      "Epoch  26  started...\n",
      "Epoch [26/150], Step [100/396], Loss: 0.2492\n",
      "Epoch [26/150], Step [200/396], Loss: 0.6251\n",
      "Epoch [26/150], Step [300/396], Loss: 0.4851\n",
      "Epoch done in  0:01:53.492717\n",
      "Epoch  27  started...\n",
      "Epoch [27/150], Step [100/396], Loss: 0.1433\n",
      "Epoch [27/150], Step [200/396], Loss: 0.3405\n",
      "Epoch [27/150], Step [300/396], Loss: 0.1962\n",
      "Epoch done in  0:01:52.847236\n",
      "Epoch  28  started...\n",
      "Epoch [28/150], Step [100/396], Loss: 0.1733\n",
      "Epoch [28/150], Step [200/396], Loss: 0.1473\n",
      "Epoch [28/150], Step [300/396], Loss: 0.3393\n",
      "Epoch done in  0:01:53.241874\n",
      "Epoch  29  started...\n",
      "Epoch [29/150], Step [100/396], Loss: 0.1353\n",
      "Epoch [29/150], Step [200/396], Loss: 0.1037\n",
      "Epoch [29/150], Step [300/396], Loss: 0.0783\n",
      "Epoch done in  0:01:52.973407\n",
      "Epoch  30  started...\n",
      "Epoch [30/150], Step [100/396], Loss: 0.5759\n",
      "Epoch [30/150], Step [200/396], Loss: 0.3957\n",
      "Epoch [30/150], Step [300/396], Loss: 0.2336\n",
      "Epoch done in  0:01:52.946079\n",
      "Epoch  31  started...\n",
      "Epoch [31/150], Step [100/396], Loss: 0.1517\n",
      "Epoch [31/150], Step [200/396], Loss: 0.2316\n",
      "Epoch [31/150], Step [300/396], Loss: 0.6128\n",
      "Epoch done in  0:01:52.635537\n",
      "Epoch  32  started...\n",
      "Epoch [32/150], Step [100/396], Loss: 0.8136\n",
      "Epoch [32/150], Step [200/396], Loss: 0.3560\n",
      "Epoch [32/150], Step [300/396], Loss: 0.1887\n",
      "Epoch done in  0:01:52.878632\n",
      "Epoch  33  started...\n",
      "Epoch [33/150], Step [100/396], Loss: 0.2419\n",
      "Epoch [33/150], Step [200/396], Loss: 0.2537\n",
      "Epoch [33/150], Step [300/396], Loss: 0.5964\n",
      "Epoch done in  0:01:53.289303\n",
      "Epoch  34  started...\n",
      "Epoch [34/150], Step [100/396], Loss: 0.0891\n",
      "Epoch [34/150], Step [200/396], Loss: 0.2601\n",
      "Epoch [34/150], Step [300/396], Loss: 0.4087\n",
      "Epoch done in  0:01:53.129022\n",
      "Epoch  35  started...\n",
      "Epoch [35/150], Step [100/396], Loss: 0.0830\n",
      "Epoch [35/150], Step [200/396], Loss: 0.1573\n",
      "Epoch [35/150], Step [300/396], Loss: 0.1717\n",
      "Epoch done in  0:01:53.624492\n",
      "Epoch  36  started...\n",
      "Epoch [36/150], Step [100/396], Loss: 0.2250\n",
      "Epoch [36/150], Step [200/396], Loss: 0.1505\n",
      "Epoch [36/150], Step [300/396], Loss: 0.1217\n",
      "Epoch done in  0:01:52.472548\n",
      "Epoch  37  started...\n",
      "Epoch [37/150], Step [100/396], Loss: 0.3429\n",
      "Epoch [37/150], Step [200/396], Loss: 0.3278\n",
      "Epoch [37/150], Step [300/396], Loss: 0.1711\n",
      "Epoch done in  0:01:52.379013\n",
      "Epoch  38  started...\n",
      "Epoch [38/150], Step [100/396], Loss: 0.4403\n",
      "Epoch [38/150], Step [200/396], Loss: 0.4493\n",
      "Epoch [38/150], Step [300/396], Loss: 0.2752\n",
      "Epoch done in  0:01:52.591954\n",
      "Epoch  39  started...\n",
      "Epoch [39/150], Step [100/396], Loss: 0.2409\n",
      "Epoch [39/150], Step [200/396], Loss: 0.8068\n",
      "Epoch [39/150], Step [300/396], Loss: 0.0833\n",
      "Epoch done in  0:01:52.045136\n",
      "Epoch  40  started...\n",
      "Epoch [40/150], Step [100/396], Loss: 0.0534\n",
      "Epoch [40/150], Step [200/396], Loss: 0.9931\n",
      "Epoch [40/150], Step [300/396], Loss: 0.1008\n",
      "Epoch done in  0:01:52.867265\n",
      "Epoch  41  started...\n",
      "Epoch [41/150], Step [100/396], Loss: 0.2345\n",
      "Epoch [41/150], Step [200/396], Loss: 0.1151\n",
      "Epoch [41/150], Step [300/396], Loss: 0.2231\n",
      "Epoch done in  0:01:52.892399\n",
      "Epoch  42  started...\n",
      "Epoch [42/150], Step [100/396], Loss: 0.2481\n",
      "Epoch [42/150], Step [200/396], Loss: 0.0236\n",
      "=> Loss reduced by:\t 0.027106687426567078\n",
      "   Saving model state\n",
      "Epoch [42/150], Step [300/396], Loss: 0.2413\n",
      "Epoch done in  0:01:53.121945\n",
      "Epoch  43  started...\n",
      "Epoch [43/150], Step [100/396], Loss: 0.3161\n",
      "Epoch [43/150], Step [200/396], Loss: 0.2279\n",
      "Epoch [43/150], Step [300/396], Loss: 0.2033\n",
      "Epoch done in  0:01:53.765484\n",
      "Epoch  44  started...\n",
      "Epoch [44/150], Step [100/396], Loss: 0.0947\n",
      "Epoch [44/150], Step [200/396], Loss: 0.1005\n",
      "Epoch [44/150], Step [300/396], Loss: 0.1888\n",
      "Epoch done in  0:01:53.330616\n",
      "Epoch  45  started...\n",
      "Epoch [45/150], Step [100/396], Loss: 0.1725\n",
      "Epoch [45/150], Step [200/396], Loss: 0.0404\n",
      "Epoch [45/150], Step [300/396], Loss: 0.0946\n",
      "Epoch done in  0:01:55.477193\n",
      "Epoch  46  started...\n",
      "Epoch [46/150], Step [100/396], Loss: 0.2468\n",
      "Epoch [46/150], Step [200/396], Loss: 0.1738\n",
      "Epoch [46/150], Step [300/396], Loss: 0.0367\n",
      "Epoch done in  0:01:56.796508\n",
      "Epoch  47  started...\n",
      "Epoch [47/150], Step [100/396], Loss: 0.1768\n",
      "Epoch [47/150], Step [200/396], Loss: 0.2863\n",
      "Epoch [47/150], Step [300/396], Loss: 0.0729\n",
      "Epoch done in  0:01:57.351030\n",
      "Epoch  48  started...\n",
      "Epoch [48/150], Step [100/396], Loss: 0.2886\n",
      "Epoch [48/150], Step [200/396], Loss: 0.0625\n",
      "Epoch [48/150], Step [300/396], Loss: 0.1073\n",
      "Epoch done in  0:01:58.050856\n",
      "Epoch  49  started...\n",
      "Epoch [49/150], Step [100/396], Loss: 0.1158\n",
      "Epoch [49/150], Step [200/396], Loss: 0.0890\n",
      "Epoch [49/150], Step [300/396], Loss: 0.3910\n",
      "Epoch done in  0:01:58.241601\n",
      "Epoch  50  started...\n",
      "Epoch [50/150], Step [100/396], Loss: 0.0696\n",
      "Epoch [50/150], Step [200/396], Loss: 0.3069\n",
      "Epoch [50/150], Step [300/396], Loss: 0.2001\n",
      "Epoch done in  0:01:58.693215\n",
      "Epoch  51  started...\n",
      "Epoch [51/150], Step [100/396], Loss: 0.0988\n",
      "Epoch [51/150], Step [200/396], Loss: 0.1227\n",
      "Epoch [51/150], Step [300/396], Loss: 0.8123\n",
      "Epoch done in  0:01:59.535355\n",
      "Epoch  52  started...\n",
      "Epoch [52/150], Step [100/396], Loss: 0.5573\n",
      "Epoch [52/150], Step [200/396], Loss: 0.0698\n",
      "Epoch [52/150], Step [300/396], Loss: 0.3191\n",
      "Epoch done in  0:01:58.117756\n",
      "Epoch  53  started...\n",
      "Epoch [53/150], Step [100/396], Loss: 0.5236\n",
      "Epoch [53/150], Step [200/396], Loss: 0.0669\n",
      "Epoch [53/150], Step [300/396], Loss: 0.2445\n",
      "Epoch done in  0:01:57.277790\n",
      "Epoch  54  started...\n",
      "Epoch [54/150], Step [100/396], Loss: 0.0897\n",
      "Epoch [54/150], Step [200/396], Loss: 0.0330\n",
      "Epoch [54/150], Step [300/396], Loss: 0.0273\n",
      "Epoch done in  0:01:59.285598\n",
      "Epoch  55  started...\n",
      "Epoch [55/150], Step [100/396], Loss: 0.0313\n",
      "Epoch [55/150], Step [200/396], Loss: 0.0406\n",
      "Epoch [55/150], Step [300/396], Loss: 0.0720\n",
      "Epoch done in  0:01:59.632593\n",
      "Epoch  56  started...\n",
      "Epoch [56/150], Step [100/396], Loss: 0.1814\n",
      "Epoch [56/150], Step [200/396], Loss: 0.1777\n",
      "Epoch [56/150], Step [300/396], Loss: 0.1634\n",
      "Epoch done in  0:02:00.292484\n",
      "Epoch  57  started...\n",
      "Epoch [57/150], Step [100/396], Loss: 0.0501\n",
      "Epoch [57/150], Step [200/396], Loss: 0.4061\n",
      "Epoch [57/150], Step [300/396], Loss: 0.1123\n",
      "Epoch done in  0:02:01.203250\n",
      "Epoch  58  started...\n",
      "Epoch [58/150], Step [100/396], Loss: 0.0128\n",
      "=> Loss reduced by:\t 0.01088152825832367\n",
      "   Saving model state\n",
      "Epoch [58/150], Step [200/396], Loss: 0.1707\n",
      "Epoch [58/150], Step [300/396], Loss: 0.1034\n",
      "Epoch done in  0:02:00.992743\n",
      "Epoch  59  started...\n",
      "Epoch [59/150], Step [100/396], Loss: 0.0977\n",
      "Epoch [59/150], Step [200/396], Loss: 0.1069\n",
      "Epoch [59/150], Step [300/396], Loss: 0.0522\n",
      "Epoch done in  0:02:00.601356\n",
      "Epoch  60  started...\n",
      "Epoch [60/150], Step [100/396], Loss: 0.0209\n",
      "Epoch [60/150], Step [200/396], Loss: 0.1908\n",
      "Epoch [60/150], Step [300/396], Loss: 0.1196\n",
      "Epoch done in  0:02:01.299314\n",
      "Epoch  61  started...\n",
      "Epoch [61/150], Step [100/396], Loss: 0.1759\n",
      "Epoch [61/150], Step [200/396], Loss: 0.0123\n",
      "=> Loss reduced by:\t 0.0004773363471031189\n",
      "   Saving model state\n",
      "Epoch [61/150], Step [300/396], Loss: 0.0144\n",
      "Epoch done in  0:02:00.676124\n",
      "Epoch  62  started...\n",
      "Epoch [62/150], Step [100/396], Loss: 0.0459\n",
      "Epoch [62/150], Step [200/396], Loss: 0.0219\n",
      "Epoch [62/150], Step [300/396], Loss: 0.0189\n",
      "Epoch done in  0:02:02.245744\n",
      "Epoch  63  started...\n",
      "Epoch [63/150], Step [100/396], Loss: 0.0692\n",
      "Epoch [63/150], Step [200/396], Loss: 0.0657\n",
      "Epoch [63/150], Step [300/396], Loss: 0.0878\n",
      "Epoch done in  0:02:01.727430\n",
      "Epoch  64  started...\n",
      "Epoch [64/150], Step [100/396], Loss: 0.0190\n",
      "Epoch [64/150], Step [200/396], Loss: 0.0495\n",
      "Epoch [64/150], Step [300/396], Loss: 0.2720\n",
      "Epoch done in  0:01:59.908623\n",
      "Epoch  65  started...\n",
      "Epoch [65/150], Step [100/396], Loss: 0.0976\n",
      "Epoch [65/150], Step [200/396], Loss: 0.0839\n",
      "Epoch [65/150], Step [300/396], Loss: 0.0562\n",
      "Epoch done in  0:01:57.578625\n",
      "Epoch  66  started...\n",
      "Epoch [66/150], Step [100/396], Loss: 0.1049\n",
      "Epoch [66/150], Step [200/396], Loss: 0.0054\n",
      "=> Loss reduced by:\t 0.006863988935947418\n",
      "   Saving model state\n",
      "Epoch [66/150], Step [300/396], Loss: 0.2185\n",
      "Epoch done in  0:01:56.173340\n",
      "Epoch  67  started...\n",
      "Epoch [67/150], Step [100/396], Loss: 0.0804\n",
      "Epoch [67/150], Step [200/396], Loss: 0.0488\n",
      "Epoch [67/150], Step [300/396], Loss: 0.1220\n",
      "Epoch done in  0:01:54.542068\n",
      "Epoch  68  started...\n",
      "Epoch [68/150], Step [100/396], Loss: 0.0895\n",
      "Epoch [68/150], Step [200/396], Loss: 0.1111\n",
      "Epoch [68/150], Step [300/396], Loss: 0.1566\n",
      "Epoch done in  0:01:53.756947\n",
      "Epoch  69  started...\n",
      "Epoch [69/150], Step [100/396], Loss: 0.0082\n",
      "Epoch [69/150], Step [200/396], Loss: 0.0749\n",
      "Epoch [69/150], Step [300/396], Loss: 0.0308\n",
      "Epoch done in  0:01:53.349150\n",
      "Epoch  70  started...\n",
      "Epoch [70/150], Step [100/396], Loss: 0.0805\n",
      "Epoch [70/150], Step [200/396], Loss: 0.1444\n",
      "Epoch [70/150], Step [300/396], Loss: 0.0910\n",
      "Epoch done in  0:01:52.851729\n",
      "Epoch  71  started...\n",
      "Epoch [71/150], Step [100/396], Loss: 0.0201\n",
      "Epoch [71/150], Step [200/396], Loss: 0.0780\n",
      "Epoch [71/150], Step [300/396], Loss: 0.0540\n",
      "Epoch done in  0:01:52.765082\n",
      "Epoch  72  started...\n",
      "Epoch [72/150], Step [100/396], Loss: 0.0075\n",
      "Epoch [72/150], Step [200/396], Loss: 0.0848\n",
      "Epoch [72/150], Step [300/396], Loss: 0.4878\n",
      "Epoch done in  0:01:53.337572\n",
      "Epoch  73  started...\n",
      "Epoch [73/150], Step [100/396], Loss: 0.2816\n",
      "Epoch [73/150], Step [200/396], Loss: 0.0611\n",
      "Epoch [73/150], Step [300/396], Loss: 0.0475\n",
      "Epoch done in  0:01:52.379184\n",
      "Epoch  74  started...\n",
      "Epoch [74/150], Step [100/396], Loss: 0.0485\n",
      "Epoch [74/150], Step [200/396], Loss: 0.3698\n",
      "Epoch [74/150], Step [300/396], Loss: 0.0473\n",
      "Epoch done in  0:01:52.100741\n",
      "Epoch  75  started...\n",
      "Epoch [75/150], Step [100/396], Loss: 0.1293\n",
      "Epoch [75/150], Step [200/396], Loss: 0.1043\n",
      "Epoch [75/150], Step [300/396], Loss: 0.1623\n",
      "Epoch done in  0:01:52.421904\n",
      "Epoch  76  started...\n",
      "Epoch [76/150], Step [100/396], Loss: 0.5520\n",
      "Epoch [76/150], Step [200/396], Loss: 0.0167\n",
      "Epoch [76/150], Step [300/396], Loss: 0.1681\n",
      "Epoch done in  0:01:52.039245\n",
      "Epoch  77  started...\n",
      "Epoch [77/150], Step [100/396], Loss: 0.1104\n",
      "Epoch [77/150], Step [200/396], Loss: 0.0311\n",
      "Epoch [77/150], Step [300/396], Loss: 0.1020\n",
      "Epoch done in  0:01:52.061710\n",
      "Epoch  78  started...\n",
      "Epoch [78/150], Step [100/396], Loss: 0.0718\n",
      "Epoch [78/150], Step [200/396], Loss: 0.0580\n",
      "Epoch [78/150], Step [300/396], Loss: 0.0193\n",
      "Epoch done in  0:01:52.813650\n",
      "Epoch  79  started...\n",
      "Epoch [79/150], Step [100/396], Loss: 0.0619\n",
      "Epoch [79/150], Step [200/396], Loss: 0.1683\n",
      "Epoch [79/150], Step [300/396], Loss: 0.0186\n",
      "Epoch done in  0:01:52.620376\n",
      "Epoch  80  started...\n",
      "Epoch [80/150], Step [100/396], Loss: 0.1255\n",
      "Epoch [80/150], Step [200/396], Loss: 0.0280\n",
      "Epoch [80/150], Step [300/396], Loss: 0.0104\n",
      "Epoch done in  0:01:52.792350\n",
      "Epoch  81  started...\n",
      "Epoch [81/150], Step [100/396], Loss: 0.0464\n",
      "Epoch [81/150], Step [200/396], Loss: 0.0742\n",
      "Epoch [81/150], Step [300/396], Loss: 0.0276\n",
      "Epoch done in  0:01:53.074571\n",
      "Epoch  82  started...\n",
      "Epoch [82/150], Step [100/396], Loss: 0.1945\n",
      "Epoch [82/150], Step [200/396], Loss: 0.0255\n",
      "Epoch [82/150], Step [300/396], Loss: 0.1307\n",
      "Epoch done in  0:01:52.640934\n",
      "Epoch  83  started...\n",
      "Epoch [83/150], Step [100/396], Loss: 0.0158\n",
      "Epoch [83/150], Step [200/396], Loss: 0.0078\n",
      "Epoch [83/150], Step [300/396], Loss: 0.0230\n",
      "Epoch done in  0:01:52.811713\n",
      "Epoch  84  started...\n",
      "Epoch [84/150], Step [100/396], Loss: 0.1383\n",
      "Epoch [84/150], Step [200/396], Loss: 0.0530\n",
      "Epoch [84/150], Step [300/396], Loss: 0.0586\n",
      "Epoch done in  0:01:52.276893\n",
      "Epoch  85  started...\n",
      "Epoch [85/150], Step [100/396], Loss: 0.0940\n",
      "Epoch [85/150], Step [200/396], Loss: 0.0137\n",
      "Epoch [85/150], Step [300/396], Loss: 0.0174\n",
      "Epoch done in  0:01:53.831892\n",
      "Epoch  86  started...\n",
      "Epoch [86/150], Step [100/396], Loss: 0.0178\n",
      "Epoch [86/150], Step [200/396], Loss: 0.0155\n",
      "Epoch [86/150], Step [300/396], Loss: 0.0090\n",
      "Epoch done in  0:01:52.895915\n",
      "Epoch  87  started...\n",
      "Epoch [87/150], Step [100/396], Loss: 0.1602\n",
      "Epoch [87/150], Step [200/396], Loss: 0.0080\n",
      "Epoch [87/150], Step [300/396], Loss: 0.0143\n",
      "Epoch done in  0:01:52.622226\n",
      "Epoch  88  started...\n",
      "Epoch [88/150], Step [100/396], Loss: 0.0262\n",
      "Epoch [88/150], Step [200/396], Loss: 0.2083\n",
      "Epoch [88/150], Step [300/396], Loss: 0.1140\n",
      "Epoch done in  0:01:53.156782\n",
      "Epoch  89  started...\n",
      "Epoch [89/150], Step [100/396], Loss: 0.0054\n",
      "=> Loss reduced by:\t 3.364682197570801e-05\n",
      "   Saving model state\n",
      "Epoch [89/150], Step [200/396], Loss: 0.1419\n",
      "Epoch [89/150], Step [300/396], Loss: 0.0795\n",
      "Epoch done in  0:01:52.898079\n",
      "Epoch  90  started...\n",
      "Epoch [90/150], Step [100/396], Loss: 0.0431\n",
      "Epoch [90/150], Step [200/396], Loss: 0.0748\n",
      "Epoch [90/150], Step [300/396], Loss: 0.0553\n",
      "Epoch done in  0:01:52.432169\n",
      "Epoch  91  started...\n",
      "Epoch [91/150], Step [100/396], Loss: 0.0575\n",
      "Epoch [91/150], Step [200/396], Loss: 0.0048\n",
      "=> Loss reduced by:\t 0.0005572140216827393\n",
      "   Saving model state\n",
      "Epoch [91/150], Step [300/396], Loss: 0.0678\n",
      "Epoch done in  0:01:52.620999\n",
      "Epoch  92  started...\n",
      "Epoch [92/150], Step [100/396], Loss: 0.0106\n",
      "Epoch [92/150], Step [200/396], Loss: 0.1930\n",
      "Epoch [92/150], Step [300/396], Loss: 0.0055\n",
      "Epoch done in  0:01:52.407575\n",
      "Epoch  93  started...\n",
      "Epoch [93/150], Step [100/396], Loss: 0.0945\n",
      "Epoch [93/150], Step [200/396], Loss: 0.0355\n",
      "Epoch [93/150], Step [300/396], Loss: 0.0668\n",
      "Epoch done in  0:01:53.053991\n",
      "Epoch  94  started...\n",
      "Epoch [94/150], Step [100/396], Loss: 0.0057\n",
      "Epoch [94/150], Step [200/396], Loss: 0.0064\n",
      "Epoch [94/150], Step [300/396], Loss: 0.1029\n",
      "Epoch done in  0:01:52.916796\n",
      "Epoch  95  started...\n",
      "Epoch [95/150], Step [100/396], Loss: 0.0536\n",
      "Epoch [95/150], Step [200/396], Loss: 0.0248\n",
      "Epoch [95/150], Step [300/396], Loss: 0.0414\n",
      "Epoch done in  0:01:52.275843\n",
      "Epoch  96  started...\n",
      "Epoch [96/150], Step [100/396], Loss: 0.0630\n",
      "Epoch [96/150], Step [200/396], Loss: 0.0189\n",
      "Epoch [96/150], Step [300/396], Loss: 0.0742\n",
      "Epoch done in  0:01:52.210983\n",
      "Epoch  97  started...\n",
      "Epoch [97/150], Step [100/396], Loss: 0.0274\n",
      "Epoch [97/150], Step [200/396], Loss: 0.2204\n",
      "Epoch [97/150], Step [300/396], Loss: 0.0312\n",
      "Epoch done in  0:01:52.565378\n",
      "Epoch  98  started...\n",
      "Epoch [98/150], Step [100/396], Loss: 0.0320\n",
      "Epoch [98/150], Step [200/396], Loss: 0.1310\n",
      "Epoch [98/150], Step [300/396], Loss: 0.0700\n",
      "Epoch done in  0:01:54.024547\n",
      "Epoch  99  started...\n",
      "Epoch [99/150], Step [100/396], Loss: 0.0068\n",
      "Epoch [99/150], Step [200/396], Loss: 0.0391\n",
      "Epoch [99/150], Step [300/396], Loss: 0.0460\n",
      "Epoch done in  0:01:52.053060\n",
      "Epoch  100  started...\n",
      "Epoch [100/150], Step [100/396], Loss: 0.0198\n",
      "Epoch [100/150], Step [200/396], Loss: 0.2123\n",
      "Epoch [100/150], Step [300/396], Loss: 0.0939\n",
      "Epoch done in  0:01:51.548689\n",
      "Epoch  101  started...\n",
      "Epoch [101/150], Step [100/396], Loss: 0.0029\n",
      "=> Loss reduced by:\t 0.0019387155771255493\n",
      "   Saving model state\n",
      "Epoch [101/150], Step [200/396], Loss: 0.0502\n",
      "Epoch [101/150], Step [300/396], Loss: 0.0412\n",
      "Epoch done in  0:01:52.871266\n",
      "Epoch  102  started...\n",
      "Epoch [102/150], Step [100/396], Loss: 0.0143\n",
      "Epoch [102/150], Step [200/396], Loss: 0.0183\n",
      "Epoch [102/150], Step [300/396], Loss: 0.0813\n",
      "Epoch done in  0:01:55.586295\n",
      "Epoch  103  started...\n",
      "Epoch [103/150], Step [100/396], Loss: 0.1684\n",
      "Epoch [103/150], Step [200/396], Loss: 0.0663\n",
      "Epoch [103/150], Step [300/396], Loss: 0.0455\n",
      "Epoch done in  0:01:54.084227\n",
      "Epoch  104  started...\n",
      "Epoch [104/150], Step [100/396], Loss: 0.0177\n",
      "Epoch [104/150], Step [200/396], Loss: 0.2851\n",
      "Epoch [104/150], Step [300/396], Loss: 0.0731\n",
      "Epoch done in  0:01:55.696666\n",
      "Epoch  105  started...\n",
      "Epoch [105/150], Step [100/396], Loss: 0.0933\n",
      "Epoch [105/150], Step [200/396], Loss: 0.0494\n",
      "Epoch [105/150], Step [300/396], Loss: 0.2441\n",
      "Epoch done in  0:01:55.174090\n",
      "Epoch  106  started...\n",
      "Epoch [106/150], Step [100/396], Loss: 0.0816\n",
      "Epoch [106/150], Step [200/396], Loss: 0.1818\n",
      "Epoch [106/150], Step [300/396], Loss: 0.0011\n",
      "=> Loss reduced by:\t 0.0017894655466079712\n",
      "   Saving model state\n",
      "Epoch done in  0:01:54.206383\n",
      "Epoch  107  started...\n",
      "Epoch [107/150], Step [100/396], Loss: 0.0818\n",
      "Epoch [107/150], Step [200/396], Loss: 0.0125\n",
      "Epoch [107/150], Step [300/396], Loss: 0.4889\n",
      "Epoch done in  0:01:56.212155\n",
      "Epoch  108  started...\n",
      "Epoch [108/150], Step [100/396], Loss: 0.1314\n",
      "Epoch [108/150], Step [200/396], Loss: 0.1460\n",
      "Epoch [108/150], Step [300/396], Loss: 0.0621\n",
      "Epoch done in  0:01:58.079133\n",
      "Epoch  109  started...\n",
      "Epoch [109/150], Step [100/396], Loss: 0.0391\n",
      "Epoch [109/150], Step [200/396], Loss: 0.0416\n",
      "Epoch [109/150], Step [300/396], Loss: 0.0781\n",
      "Epoch done in  0:01:58.511277\n",
      "Epoch  110  started...\n",
      "Epoch [110/150], Step [100/396], Loss: 0.0112\n",
      "Epoch [110/150], Step [200/396], Loss: 0.0762\n",
      "Epoch [110/150], Step [300/396], Loss: 0.2185\n",
      "Epoch done in  0:01:59.047598\n",
      "Epoch  111  started...\n",
      "Epoch [111/150], Step [100/396], Loss: 0.0363\n",
      "Epoch [111/150], Step [200/396], Loss: 0.1254\n",
      "Epoch [111/150], Step [300/396], Loss: 0.0540\n",
      "Epoch done in  0:01:59.170366\n",
      "Epoch  112  started...\n",
      "Epoch [112/150], Step [100/396], Loss: 0.1777\n",
      "Epoch [112/150], Step [200/396], Loss: 0.3335\n",
      "Epoch [112/150], Step [300/396], Loss: 0.0384\n",
      "Epoch done in  0:01:59.990848\n",
      "Epoch  113  started...\n",
      "Epoch [113/150], Step [100/396], Loss: 0.0107\n",
      "Epoch [113/150], Step [200/396], Loss: 0.0878\n",
      "Epoch [113/150], Step [300/396], Loss: 0.0420\n",
      "Epoch done in  0:02:00.024396\n",
      "Epoch  114  started...\n",
      "Epoch [114/150], Step [100/396], Loss: 0.0139\n",
      "Epoch [114/150], Step [200/396], Loss: 0.0188\n",
      "Epoch [114/150], Step [300/396], Loss: 0.0033\n",
      "Epoch done in  0:01:59.633036\n",
      "Epoch  115  started...\n",
      "Epoch [115/150], Step [100/396], Loss: 0.0545\n",
      "Epoch [115/150], Step [200/396], Loss: 0.0682\n",
      "Epoch [115/150], Step [300/396], Loss: 0.0640\n",
      "Epoch done in  0:02:00.637021\n",
      "Epoch  116  started...\n",
      "Epoch [116/150], Step [100/396], Loss: 0.0152\n",
      "Epoch [116/150], Step [200/396], Loss: 0.0197\n",
      "Epoch [116/150], Step [300/396], Loss: 0.1268\n",
      "Epoch done in  0:01:59.666991\n",
      "Epoch  117  started...\n",
      "Epoch [117/150], Step [100/396], Loss: 0.0561\n",
      "Epoch [117/150], Step [200/396], Loss: 0.2566\n",
      "Epoch [117/150], Step [300/396], Loss: 0.0456\n",
      "Epoch done in  0:01:56.855483\n",
      "Epoch  118  started...\n",
      "Epoch [118/150], Step [100/396], Loss: 0.0676\n",
      "Epoch [118/150], Step [200/396], Loss: 0.0100\n",
      "Epoch [118/150], Step [300/396], Loss: 0.0331\n",
      "Epoch done in  0:01:54.571018\n",
      "Epoch  119  started...\n",
      "Epoch [119/150], Step [100/396], Loss: 0.0220\n",
      "Epoch [119/150], Step [200/396], Loss: 0.0162\n",
      "Epoch [119/150], Step [300/396], Loss: 0.1673\n",
      "Epoch done in  0:01:55.481611\n",
      "Epoch  120  started...\n",
      "Epoch [120/150], Step [100/396], Loss: 0.0045\n",
      "Epoch [120/150], Step [200/396], Loss: 0.0277\n",
      "Epoch [120/150], Step [300/396], Loss: 0.0682\n",
      "Epoch done in  0:01:54.638861\n",
      "Epoch  121  started...\n",
      "Epoch [121/150], Step [100/396], Loss: 0.0805\n",
      "Epoch [121/150], Step [200/396], Loss: 0.0005\n",
      "=> Loss reduced by:\t 0.0006348937749862671\n",
      "   Saving model state\n",
      "Epoch [121/150], Step [300/396], Loss: 0.0331\n",
      "Epoch done in  0:01:55.610869\n",
      "Epoch  122  started...\n",
      "Epoch [122/150], Step [100/396], Loss: 0.0808\n",
      "Epoch [122/150], Step [200/396], Loss: 0.0130\n",
      "Epoch [122/150], Step [300/396], Loss: 0.0149\n",
      "Epoch done in  0:01:55.072701\n",
      "Epoch  123  started...\n",
      "Epoch [123/150], Step [100/396], Loss: 0.1006\n",
      "Epoch [123/150], Step [200/396], Loss: 0.0670\n",
      "Epoch [123/150], Step [300/396], Loss: 0.1156\n",
      "Epoch done in  0:01:56.132019\n",
      "Epoch  124  started...\n",
      "Epoch [124/150], Step [100/396], Loss: 0.0013\n",
      "Epoch [124/150], Step [200/396], Loss: 0.1159\n",
      "Epoch [124/150], Step [300/396], Loss: 0.2435\n",
      "Epoch done in  0:01:56.017267\n",
      "Epoch  125  started...\n",
      "Epoch [125/150], Step [100/396], Loss: 0.0921\n",
      "Epoch [125/150], Step [200/396], Loss: 0.0611\n",
      "Epoch [125/150], Step [300/396], Loss: 0.1214\n",
      "Epoch done in  0:01:56.421778\n",
      "Epoch  126  started...\n",
      "Epoch [126/150], Step [100/396], Loss: 0.0408\n",
      "Epoch [126/150], Step [200/396], Loss: 0.0938\n",
      "Epoch [126/150], Step [300/396], Loss: 0.1145\n",
      "Epoch done in  0:01:55.934986\n",
      "Epoch  127  started...\n",
      "Epoch [127/150], Step [100/396], Loss: 0.2892\n",
      "Epoch [127/150], Step [200/396], Loss: 0.0036\n",
      "Epoch [127/150], Step [300/396], Loss: 0.0052\n",
      "Epoch done in  0:01:56.508567\n",
      "Epoch  128  started...\n",
      "Epoch [128/150], Step [100/396], Loss: 0.0207\n",
      "Epoch [128/150], Step [200/396], Loss: 0.0987\n",
      "Epoch [128/150], Step [300/396], Loss: 0.0642\n",
      "Epoch done in  0:01:57.006484\n",
      "Epoch  129  started...\n",
      "Epoch [129/150], Step [100/396], Loss: 0.0424\n",
      "Epoch [129/150], Step [200/396], Loss: 0.0063\n",
      "Epoch [129/150], Step [300/396], Loss: 0.2059\n",
      "Epoch done in  0:01:57.432218\n",
      "Epoch  130  started...\n",
      "Epoch [130/150], Step [100/396], Loss: 0.0580\n",
      "Epoch [130/150], Step [200/396], Loss: 0.2178\n",
      "Epoch [130/150], Step [300/396], Loss: 0.1214\n",
      "Epoch done in  0:01:56.561541\n",
      "Epoch  131  started...\n",
      "Epoch [131/150], Step [100/396], Loss: 0.0137\n",
      "Epoch [131/150], Step [200/396], Loss: 0.0003\n",
      "=> Loss reduced by:\t 0.0001490563154220581\n",
      "   Saving model state\n",
      "Epoch [131/150], Step [300/396], Loss: 0.0282\n",
      "Epoch done in  0:01:56.665570\n",
      "Epoch  132  started...\n",
      "Epoch [132/150], Step [100/396], Loss: 0.0540\n",
      "Epoch [132/150], Step [200/396], Loss: 0.0234\n",
      "Epoch [132/150], Step [300/396], Loss: 0.0843\n",
      "Epoch done in  0:01:56.639826\n",
      "Epoch  133  started...\n",
      "Epoch [133/150], Step [100/396], Loss: 0.2124\n",
      "Epoch [133/150], Step [200/396], Loss: 0.0038\n",
      "Epoch [133/150], Step [300/396], Loss: 0.0003\n",
      "Epoch done in  0:01:56.606141\n",
      "Epoch  134  started...\n",
      "Epoch [134/150], Step [100/396], Loss: 0.0216\n",
      "Epoch [134/150], Step [200/396], Loss: 0.0623\n",
      "Epoch [134/150], Step [300/396], Loss: 0.0435\n",
      "Epoch done in  0:01:56.072843\n",
      "Epoch  135  started...\n",
      "Epoch [135/150], Step [100/396], Loss: 0.0049\n",
      "Epoch [135/150], Step [200/396], Loss: 0.1226\n",
      "Epoch [135/150], Step [300/396], Loss: 0.0136\n",
      "Epoch done in  0:01:56.910564\n",
      "Epoch  136  started...\n",
      "Epoch [136/150], Step [100/396], Loss: 0.0388\n",
      "Epoch [136/150], Step [200/396], Loss: 0.0481\n",
      "Epoch [136/150], Step [300/396], Loss: 0.0615\n",
      "Epoch done in  0:01:57.821187\n",
      "Epoch  137  started...\n",
      "Epoch [137/150], Step [100/396], Loss: 0.0440\n",
      "Epoch [137/150], Step [200/396], Loss: 0.0582\n",
      "Epoch [137/150], Step [300/396], Loss: 0.0040\n",
      "Epoch done in  0:01:56.806710\n",
      "Epoch  138  started...\n",
      "Epoch [138/150], Step [100/396], Loss: 0.0464\n",
      "Epoch [138/150], Step [200/396], Loss: 0.0068\n",
      "Epoch [138/150], Step [300/396], Loss: 0.0135\n",
      "Epoch done in  0:01:55.686256\n",
      "Epoch  139  started...\n",
      "Epoch [139/150], Step [100/396], Loss: 0.1108\n",
      "Epoch [139/150], Step [200/396], Loss: 0.0007\n",
      "Epoch [139/150], Step [300/396], Loss: 0.0014\n",
      "Epoch done in  0:01:56.337399\n",
      "Epoch  140  started...\n",
      "Epoch [140/150], Step [100/396], Loss: 0.0832\n",
      "Epoch [140/150], Step [200/396], Loss: 0.0009\n",
      "Epoch [140/150], Step [300/396], Loss: 0.0728\n",
      "Epoch done in  0:01:56.280527\n",
      "Epoch  141  started...\n",
      "Epoch [141/150], Step [100/396], Loss: 0.1150\n",
      "Epoch [141/150], Step [200/396], Loss: 0.3402\n",
      "Epoch [141/150], Step [300/396], Loss: 0.0189\n",
      "Epoch done in  0:01:55.887268\n",
      "Epoch  142  started...\n",
      "Epoch [142/150], Step [100/396], Loss: 0.0121\n",
      "Epoch [142/150], Step [200/396], Loss: 0.0606\n",
      "Epoch [142/150], Step [300/396], Loss: 0.0662\n",
      "Epoch done in  0:01:54.800587\n",
      "Epoch  143  started...\n",
      "Epoch [143/150], Step [100/396], Loss: 0.0032\n",
      "Epoch [143/150], Step [200/396], Loss: 0.0720\n",
      "Epoch [143/150], Step [300/396], Loss: 0.0600\n",
      "Epoch done in  0:01:54.500231\n",
      "Epoch  144  started...\n",
      "Epoch [144/150], Step [100/396], Loss: 0.0728\n",
      "Epoch [144/150], Step [200/396], Loss: 0.0324\n",
      "Epoch [144/150], Step [300/396], Loss: 0.0017\n",
      "Epoch done in  0:01:54.695659\n",
      "Epoch  145  started...\n",
      "Epoch [145/150], Step [100/396], Loss: 0.0025\n",
      "Epoch [145/150], Step [200/396], Loss: 0.0753\n",
      "Epoch [145/150], Step [300/396], Loss: 0.0161\n",
      "Epoch done in  0:01:55.023197\n",
      "Epoch  146  started...\n",
      "Epoch [146/150], Step [100/396], Loss: 0.0235\n",
      "Epoch [146/150], Step [200/396], Loss: 0.0055\n",
      "Epoch [146/150], Step [300/396], Loss: 0.0022\n",
      "Epoch done in  0:01:55.967227\n",
      "Epoch  147  started...\n",
      "Epoch [147/150], Step [100/396], Loss: 0.1723\n",
      "Epoch [147/150], Step [200/396], Loss: 0.0055\n",
      "Epoch [147/150], Step [300/396], Loss: 0.0498\n",
      "Epoch done in  0:01:55.438640\n",
      "Epoch  148  started...\n",
      "Epoch [148/150], Step [100/396], Loss: 0.0006\n",
      "Epoch [148/150], Step [200/396], Loss: 0.0531\n",
      "Epoch [148/150], Step [300/396], Loss: 0.0086\n",
      "Epoch done in  0:01:55.389183\n",
      "Epoch  149  started...\n",
      "Epoch [149/150], Step [100/396], Loss: 0.0952\n",
      "Epoch [149/150], Step [200/396], Loss: 0.0214\n",
      "Epoch [149/150], Step [300/396], Loss: 0.0020\n",
      "Epoch done in  0:01:54.099867\n",
      "Epoch  150  started...\n",
      "Epoch [150/150], Step [100/396], Loss: 0.0676\n",
      "Epoch [150/150], Step [200/396], Loss: 0.0427\n",
      "Epoch [150/150], Step [300/396], Loss: 0.0676\n",
      "Epoch done in  0:01:53.716894\n",
      "Training done in  4:48:04.490262\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "timestamp_train_start = time.time()\n",
    "\n",
    "loss_hist = []\n",
    "best_loss = 9\n",
    "total_step = len(loader_train)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    timestamp_epoch_start = time.time()\n",
    "    print(\"Epoch \", epoch+1,\" started...\")\n",
    "    for i, (images, labels) in enumerate(loader_train):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "                \n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "       \n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))    \n",
    "             # Checkpointing\n",
    "            loss_hist.append(float(loss.item())) #add current loss value.\n",
    "            save_checkpoint(model.state_dict(),float(loss.item()))\n",
    "                    \n",
    "    timestamp_epoch_end = time.time()\n",
    "    print(\"Epoch done in \",str(datetime.timedelta(seconds=(timestamp_epoch_end - timestamp_epoch_start))))\n",
    "    \n",
    "timestamp_train_end = time.time()\n",
    "print(\"Training done in \",str(datetime.timedelta(seconds=(timestamp_train_end - timestamp_train_start))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the test images: 39.88621997471555 %\n"
     ]
    }
   ],
   "source": [
    "#load the best model\n",
    "model.load_state_dict(torch.load(\"state_dict.dct\"))\n",
    "\n",
    "model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in loader_valid:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "          \n",
    "    print('Test Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n",
    "\n",
    "# Save the model checkpoint\n",
    "torch.save(model.state_dict(), 'final_state.dct')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total execution time:  4:48:49.258486\n"
     ]
    }
   ],
   "source": [
    "timestamp_exec_end = time.time()\n",
    "print(\"Total execution time: \",str(datetime.timedelta(seconds=(timestamp_exec_end - timestamp_exec_start))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}