pytorch_ipynb/cnn/cnn-vgg16.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "convnet-vgg16.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {
        "height": "calc(100% - 180px)",
        "left": "10px",
        "top": "150px",
        "width": "371px"
      },
      "toc_section_display": true,
      "toc_window_display": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "89bc4ee8713e475e8c864c37d47a1157": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6afe1028d4b34c33b30838bfe457643a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a4fd815c383c4c248fbba3907886849b",
              "IPY_MODEL_a0d66010642843d0a98667cdf44cd822"
            ]
          }
        },
        "6afe1028d4b34c33b30838bfe457643a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4fd815c383c4c248fbba3907886849b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_63f906dfb53a47f0bda6128c7942df7a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ecee8283a0b471a8e3738c931808733"
          }
        },
        "a0d66010642843d0a98667cdf44cd822": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_344bbe8253204e0a841ad1cca461e4e6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:30&lt;00:00, 17391083.85it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_913cc65836c744218e95671f7191c798"
          }
        },
        "63f906dfb53a47f0bda6128c7942df7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9ecee8283a0b471a8e3738c931808733": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "344bbe8253204e0a841ad1cca461e4e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "913cc65836c744218e95671f7191c798": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "toc": true,
        "id": "NGk_978ceo1v",
        "colab_type": "text"
      },
      "source": [
        "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
        "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Model-Zoo----Convolutional-Neural-Network-(VGG16)\" data-toc-modified-id=\"Model-Zoo----Convolutional-Neural-Network-(VGG16)-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Model Zoo -- Convolutional Neural Network (VGG16)</a></span><ul class=\"toc-item\"><li><span><a href=\"#Imports\" data-toc-modified-id=\"Imports-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Imports</a></span></li><li><span><a href=\"#Settings-and-Dataset\" data-toc-modified-id=\"Settings-and-Dataset-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Settings and Dataset</a></span></li><li><span><a href=\"#Model\" data-toc-modified-id=\"Model-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Model</a></span></li><li><span><a href=\"#Training\" data-toc-modified-id=\"Training-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Training</a></span></li><li><span><a href=\"#Evaluation\" data-toc-modified-id=\"Evaluation-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Evaluation</a></span></li></ul></li></ul></div>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UEBilEjLj5wY"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GujRdBvQerJt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GOzuY8Yvj5wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "2b327b85-1f13-4ebc-80c0-b75db8dbba54"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MEu9MiOxj5wk"
      },
      "source": [
        "- Runs on CPU (not recommended here) or GPU (if available)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rH4XmErYj5wm"
      },
      "source": [
        "# Model Zoo -- Convolutional Neural Network (VGG16)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MkoGLH_Tj5wn"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ORj09gnrj5wp",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "PvgJ_0i7j5wt"
      },
      "source": [
        "## Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NnT0sZIwj5wu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 135,
          "referenced_widgets": [
            "89bc4ee8713e475e8c864c37d47a1157",
            "6afe1028d4b34c33b30838bfe457643a",
            "a4fd815c383c4c248fbba3907886849b",
            "a0d66010642843d0a98667cdf44cd822",
            "63f906dfb53a47f0bda6128c7942df7a",
            "9ecee8283a0b471a8e3738c931808733",
            "344bbe8253204e0a841ad1cca461e4e6",
            "913cc65836c744218e95671f7191c798"
          ]
        },
        "outputId": "04757a76-f16f-429b-a1d1-72427b6c0308"
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Device\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print('Device:', DEVICE)\n",
        "\n",
        "# Hyperparameters\n",
        "random_seed = 1\n",
        "learning_rate = 0.001\n",
        "num_epochs = 10\n",
        "batch_size = 128\n",
        "\n",
        "# Architecture\n",
        "num_features = 784\n",
        "num_classes = 10\n",
        "\n",
        "\n",
        "##########################\n",
        "### MNIST DATASET\n",
        "##########################\n",
        "\n",
        "# Note transforms.ToTensor() scales input images\n",
        "# to 0-1 range\n",
        "train_dataset = datasets.CIFAR10(root='data', \n",
        "                                 train=True, \n",
        "                                 transform=transforms.ToTensor(),\n",
        "                                 download=True)\n",
        "\n",
        "test_dataset = datasets.CIFAR10(root='data', \n",
        "                                train=False, \n",
        "                                transform=transforms.ToTensor())\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=batch_size, \n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=batch_size, \n",
        "                         shuffle=False)\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Device: cuda\n",
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "89bc4ee8713e475e8c864c37d47a1157",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/cifar-10-python.tar.gz to data\n",
            "Image batch dimensions: torch.Size([128, 3, 32, 32])\n",
            "Image label dimensions: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I6hghKPxj5w0"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_lza9t_uj5w1",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "\n",
        "class VGG16(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, num_features, num_classes):\n",
        "        super(VGG16, self).__init__()\n",
        "        \n",
        "        # calculate same padding:\n",
        "        # (w - k + 2*p)/s + 1 = o\n",
        "        # => p = (s(o-1) - w + k)/2\n",
        "        \n",
        "        self.block_1 = nn.Sequential(\n",
        "                nn.Conv2d(in_channels=3,\n",
        "                          out_channels=64,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          # (1(32-1)- 32 + 3)/2 = 1\n",
        "                          padding=1), \n",
        "                nn.ReLU(),\n",
        "                nn.Conv2d(in_channels=64,\n",
        "                          out_channels=64,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),\n",
        "                nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                             stride=(2, 2))\n",
        "        )\n",
        "        \n",
        "        self.block_2 = nn.Sequential(\n",
        "                nn.Conv2d(in_channels=64,\n",
        "                          out_channels=128,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),\n",
        "                nn.Conv2d(in_channels=128,\n",
        "                          out_channels=128,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),\n",
        "                nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                             stride=(2, 2))\n",
        "        )\n",
        "        \n",
        "        self.block_3 = nn.Sequential(        \n",
        "                nn.Conv2d(in_channels=128,\n",
        "                          out_channels=256,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),\n",
        "                nn.Conv2d(in_channels=256,\n",
        "                          out_channels=256,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),        \n",
        "                nn.Conv2d(in_channels=256,\n",
        "                          out_channels=256,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),\n",
        "                nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                             stride=(2, 2))\n",
        "        )\n",
        "        \n",
        "          \n",
        "        self.block_4 = nn.Sequential(   \n",
        "                nn.Conv2d(in_channels=256,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),        \n",
        "                nn.Conv2d(in_channels=512,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),        \n",
        "                nn.Conv2d(in_channels=512,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),            \n",
        "                nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                             stride=(2, 2))\n",
        "        )\n",
        "        \n",
        "        self.block_5 = nn.Sequential(\n",
        "                nn.Conv2d(in_channels=512,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),            \n",
        "                nn.Conv2d(in_channels=512,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),            \n",
        "                nn.Conv2d(in_channels=512,\n",
        "                          out_channels=512,\n",
        "                          kernel_size=(3, 3),\n",
        "                          stride=(1, 1),\n",
        "                          padding=1),\n",
        "                nn.ReLU(),    \n",
        "                nn.MaxPool2d(kernel_size=(2, 2),\n",
        "                             stride=(2, 2))             \n",
        "        )\n",
        "            \n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(512, 4096),\n",
        "            nn.ReLU(True),\n",
        "            #nn.Dropout(p=0.5),\n",
        "            nn.Linear(4096, 4096),\n",
        "            nn.ReLU(True),\n",
        "            #nn.Dropout(p=0.5),\n",
        "            nn.Linear(4096, num_classes),\n",
        "        )\n",
        "            \n",
        "        for m in self.modules():\n",
        "            if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):\n",
        "                nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='relu')\n",
        "                if m.bias is not None:\n",
        "                    m.bias.detach().zero_()\n",
        "                    \n",
        "        #self.avgpool = nn.AdaptiveAvgPool2d((7, 7))\n",
        "        \n",
        "        \n",
        "    def forward(self, x):\n",
        "\n",
        "        x = self.block_1(x)\n",
        "        x = self.block_2(x)\n",
        "        x = self.block_3(x)\n",
        "        x = self.block_4(x)\n",
        "        x = self.block_5(x)\n",
        "        #x = self.avgpool(x)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        logits = self.classifier(x)\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "\n",
        "        return logits, probas\n",
        "\n",
        "    \n",
        "torch.manual_seed(random_seed)\n",
        "model = VGG16(num_features=num_features,\n",
        "              num_classes=num_classes)\n",
        "\n",
        "model = model.to(DEVICE)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NLMkTZgIfDxz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "outputId": "80366327-9cc8-4769-dd65-8649b8c14f47"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([12, 3, 32, 32]).to(DEVICE))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7fb48f738b38>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2686pt\" height=\"200pt\"\n viewBox=\"0.00 0.00 2686.00 200.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 164)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-164 2614,-164 2614,36 -72,36\"/>\n<!-- /outputs/37 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/37</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"248,-44 178,-44 178,-8 248,-8 248,-44\"/>\n<text text-anchor=\"start\" x=\"186\" y=\"-23\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- 7040190244931630401 -->\n<g id=\"node17\" class=\"node\">\n<title>7040190244931630401</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"426,-48 342,-48 342,-4 426,-4 426,-48\"/>\n<text text-anchor=\"start\" x=\"350\" y=\"-32\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"411\" y=\"-11\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x2</text>\n</g>\n<!-- /outputs/37&#45;&gt;7040190244931630401 -->\n<g id=\"edge12\" class=\"edge\">\n<title>/outputs/37&#45;&gt;7040190244931630401</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M248.017,-26C272.1037,-26 304.5182,-26 331.6582,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"331.7732,-29.5001 341.7732,-26 331.7732,-22.5001 331.7732,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"295\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x64x16x16</text>\n</g>\n<!-- /outputs/42 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/42</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"596,-44 526,-44 526,-8 596,-8 596,-44\"/>\n<text text-anchor=\"start\" x=\"534\" y=\"-23\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- 18445400969226953810 -->\n<g id=\"node19\" class=\"node\">\n<title>18445400969226953810</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"769,-48 685,-48 685,-4 769,-4 769,-48\"/>\n<text text-anchor=\"start\" x=\"693\" y=\"-32\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"754\" y=\"-11\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/42&#45;&gt;18445400969226953810 -->\n<g id=\"edge16\" class=\"edge\">\n<title>/outputs/42&#45;&gt;18445400969226953810</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M596.2043,-26C618.9563,-26 648.9768,-26 674.518,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.7892,-29.5001 684.7892,-26 674.7891,-22.5001 674.7892,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"640.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x128x8x8</text>\n</g>\n<!-- /outputs/49 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/49</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"928,-44 858,-44 858,-8 928,-8 928,-44\"/>\n<text text-anchor=\"start\" x=\"866\" y=\"-23\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- 14362121739051703677 -->\n<g id=\"node20\" class=\"node\">\n<title>14362121739051703677</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1101,-48 1017,-48 1017,-4 1101,-4 1101,-48\"/>\n<text text-anchor=\"start\" x=\"1025\" y=\"-32\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"1086\" y=\"-11\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/49&#45;&gt;14362121739051703677 -->\n<g id=\"edge18\" class=\"edge\">\n<title>/outputs/49&#45;&gt;14362121739051703677</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M928.2043,-26C950.9563,-26 980.9768,-26 1006.518,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1006.7892,-29.5001 1016.7892,-26 1006.7891,-22.5001 1006.7892,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"972.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x256x4x4</text>\n</g>\n<!-- /outputs/56 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/56</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1260,-44 1190,-44 1190,-8 1260,-8 1260,-44\"/>\n<text text-anchor=\"start\" x=\"1198\" y=\"-23\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- 9442230734797139557 -->\n<g id=\"node21\" class=\"node\">\n<title>9442230734797139557</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1433,-48 1349,-48 1349,-4 1433,-4 1433,-48\"/>\n<text text-anchor=\"start\" x=\"1357\" y=\"-32\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"1418\" y=\"-11\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/56&#45;&gt;9442230734797139557 -->\n<g id=\"edge20\" class=\"edge\">\n<title>/outputs/56&#45;&gt;9442230734797139557</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1260.2043,-26C1282.9563,-26 1312.9768,-26 1338.518,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1338.7892,-29.5001 1348.7892,-26 1338.7891,-22.5001 1338.7892,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"1304.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512x2x2</text>\n</g>\n<!-- /outputs/63 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/63</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1592,-44 1522,-44 1522,-8 1592,-8 1592,-44\"/>\n<text text-anchor=\"start\" x=\"1530\" y=\"-23\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool2x2</text>\n</g>\n<!-- /outputs/64 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/64</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1735,-74 1681,-74 1681,-38 1735,-38 1735,-74\"/>\n<text text-anchor=\"start\" x=\"1696\" y=\"-53\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/63&#45;&gt;/outputs/64 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/63&#45;&gt;/outputs/64</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1592.022,-32.958C1615.6541,-37.6531 1646.7981,-43.8407 1670.8618,-48.6215\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1670.2084,-52.0601 1680.6988,-50.5759 1671.5725,-45.1943 1670.2084,-52.0601\"/>\n<text text-anchor=\"middle\" x=\"1636.5\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512x1x1</text>\n</g>\n<!-- /outputs/71 -->\n<g id=\"node13\" class=\"node\">\n<title>/outputs/71</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2158,-40 2104,-40 2104,-4 2158,-4 2158,-40\"/>\n<text text-anchor=\"start\" x=\"2114\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/63&#45;&gt;/outputs/71 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/63&#45;&gt;/outputs/71</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1592.2796,-20.133C1622.8108,-15.5667 1668.1933,-10 1708,-10 1708,-10 1708,-10 2040,-10 2057.6472,-10 2077.0554,-12.2456 2093.3988,-14.7939\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2093.2092,-18.3101 2103.6468,-16.4917 2094.3534,-11.4043 2093.2092,-18.3101\"/>\n<text text-anchor=\"middle\" x=\"1870.5\" y=\"-13\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512x1x1</text>\n</g>\n<!-- /outputs/66 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/66</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1826,-74 1772,-74 1772,-38 1826,-38 1826,-74\"/>\n<text text-anchor=\"start\" x=\"1785\" y=\"-53\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Gather</text>\n</g>\n<!-- /outputs/64&#45;&gt;/outputs/66 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/64&#45;&gt;/outputs/66</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1735.303,-56C1743.5813,-56 1752.8217,-56 1761.6405,-56\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1761.7163,-59.5001 1771.7163,-56 1761.7162,-52.5001 1761.7163,-59.5001\"/>\n</g>\n<!-- /outputs/65 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/65</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1735,-128 1681,-128 1681,-92 1735,-92 1735,-128\"/>\n<text text-anchor=\"start\" x=\"1690\" y=\"-107\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/65&#45;&gt;/outputs/66 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/65&#45;&gt;/outputs/66</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1735.303,-93.7982C1744.0312,-88.6189 1753.829,-82.8048 1763.0744,-77.3185\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1764.9025,-80.3035 1771.7163,-72.1903 1761.3303,-74.2836 1764.9025,-80.3035\"/>\n</g>\n<!-- /outputs/68 -->\n<g id=\"node10\" class=\"node\">\n<title>/outputs/68</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1976,-74 1915,-74 1915,-38 1976,-38 1976,-74\"/>\n<text text-anchor=\"start\" x=\"1923.5\" y=\"-53\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/66&#45;&gt;/outputs/68 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/66&#45;&gt;/outputs/68</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1826.2122,-56C1848.223,-56 1879.5605,-56 1904.5927,-56\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1904.8892,-59.5001 1914.8891,-56 1904.8891,-52.5001 1904.8892,-59.5001\"/>\n</g>\n<!-- /outputs/67 -->\n<g id=\"node9\" class=\"node\">\n<title>/outputs/67</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1826,-128 1772,-128 1772,-92 1826,-92 1826,-128\"/>\n<text text-anchor=\"start\" x=\"1781\" y=\"-107\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/69 -->\n<g id=\"node11\" class=\"node\">\n<title>/outputs/69</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1976,-128 1915,-128 1915,-92 1976,-92 1976,-128\"/>\n<text text-anchor=\"start\" x=\"1923.5\" y=\"-107\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/67&#45;&gt;/outputs/69 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/67&#45;&gt;/outputs/69</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1826.2122,-110C1848.223,-110 1879.5605,-110 1904.5927,-110\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1904.8892,-113.5001 1914.8891,-110 1904.8891,-106.5001 1904.8892,-113.5001\"/>\n</g>\n<!-- /outputs/70 -->\n<g id=\"node12\" class=\"node\">\n<title>/outputs/70</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2067,-74 2013,-74 2013,-38 2067,-38 2067,-74\"/>\n<text text-anchor=\"start\" x=\"2025\" y=\"-53\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/68&#45;&gt;/outputs/70 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/68&#45;&gt;/outputs/70</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1976.1814,-56C1984.7033,-56 1994.0485,-56 2002.8957,-56\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2002.9735,-59.5001 2012.9735,-56 2002.9734,-52.5001 2002.9735,-59.5001\"/>\n</g>\n<!-- /outputs/69&#45;&gt;/outputs/70 -->\n<g id=\"edge8\" class=\"edge\">\n<title>/outputs/69&#45;&gt;/outputs/70</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1976.1814,-92.4677C1985.0738,-87.3864 1994.8627,-81.7927 2004.0467,-76.5447\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2006.0275,-79.444 2012.9735,-71.4437 2002.5545,-73.3663 2006.0275,-79.444\"/>\n</g>\n<!-- /outputs/70&#45;&gt;/outputs/71 -->\n<g id=\"edge9\" class=\"edge\">\n<title>/outputs/70&#45;&gt;/outputs/71</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2067.303,-45.7989C2075.7612,-42.6387 2085.224,-39.1031 2094.215,-35.7438\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2095.5738,-38.9726 2103.7163,-32.1939 2093.1237,-32.4153 2095.5738,-38.9726\"/>\n</g>\n<!-- 1113677088271459478 -->\n<g id=\"node18\" class=\"node\">\n<title>1113677088271459478</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2298,-44 2226,-44 2226,0 2298,0 2298,-44\"/>\n<text text-anchor=\"start\" x=\"2234\" y=\"-28\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear &gt; Relu</text>\n<text text-anchor=\"start\" x=\"2283\" y=\"-7\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x2</text>\n</g>\n<!-- /outputs/71&#45;&gt;1113677088271459478 -->\n<g id=\"edge14\" class=\"edge\">\n<title>/outputs/71&#45;&gt;1113677088271459478</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2158.1433,-22C2174.787,-22 2196.5325,-22 2215.8223,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2215.941,-25.5001 2225.941,-22 2215.9409,-18.5001 2215.941,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"2192\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512</text>\n</g>\n<!-- /outputs/76 -->\n<g id=\"node14\" class=\"node\">\n<title>/outputs/76</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2425,-40 2371,-40 2371,-4 2425,-4 2425,-40\"/>\n<text text-anchor=\"start\" x=\"2385\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/77 -->\n<g id=\"node15\" class=\"node\">\n<title>/outputs/77</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2542,-40 2488,-40 2488,-4 2542,-4 2542,-40\"/>\n<text text-anchor=\"start\" x=\"2498\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/76&#45;&gt;/outputs/77 -->\n<g id=\"edge10\" class=\"edge\">\n<title>/outputs/76&#45;&gt;/outputs/77</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2425.1362,-22C2440.6773,-22 2460.4112,-22 2477.4253,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2477.6057,-25.5001 2487.6056,-22 2477.6056,-18.5001 2477.6057,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"2456.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x10</text>\n</g>\n<!-- 4010001316707237263 -->\n<g id=\"node16\" class=\"node\">\n<title>4010001316707237263</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"84,-48 0,-48 0,-4 84,-4 84,-48\"/>\n<text text-anchor=\"start\" x=\"8\" y=\"-32\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"69\" y=\"-11\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x2</text>\n</g>\n<!-- 4010001316707237263&#45;&gt;/outputs/37 -->\n<g id=\"edge11\" class=\"edge\">\n<title>4010001316707237263&#45;&gt;/outputs/37</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M84.2697,-26C109.5651,-26 141.7113,-26 167.5519,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"167.8392,-29.5001 177.8391,-26 167.8391,-22.5001 167.8392,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"131\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x64x32x32</text>\n</g>\n<!-- 7040190244931630401&#45;&gt;/outputs/42 -->\n<g id=\"edge13\" class=\"edge\">\n<title>7040190244931630401&#45;&gt;/outputs/42</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M426.3959,-26C453.3301,-26 488.1688,-26 515.6543,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"515.7966,-29.5001 525.7965,-26 515.7965,-22.5001 515.7966,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"476\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x128x16x16</text>\n</g>\n<!-- 1113677088271459478&#45;&gt;/outputs/76 -->\n<g id=\"edge15\" class=\"edge\">\n<title>1113677088271459478&#45;&gt;/outputs/76</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2298.0884,-22C2317.4327,-22 2341.2329,-22 2360.7511,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2360.8091,-25.5001 2370.8091,-22 2360.809,-18.5001 2360.8091,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"2334.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x4096</text>\n</g>\n<!-- 18445400969226953810&#45;&gt;/outputs/49 -->\n<g id=\"edge17\" class=\"edge\">\n<title>18445400969226953810&#45;&gt;/outputs/49</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M769.3183,-26C793.2728,-26 823.2334,-26 847.6669,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"847.7664,-29.5001 857.7663,-26 847.7663,-22.5001 847.7664,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"813.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x256x8x8</text>\n</g>\n<!-- 14362121739051703677&#45;&gt;/outputs/56 -->\n<g id=\"edge19\" class=\"edge\">\n<title>14362121739051703677&#45;&gt;/outputs/56</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1101.3183,-26C1125.2728,-26 1155.2334,-26 1179.6669,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1179.7664,-29.5001 1189.7663,-26 1179.7663,-22.5001 1179.7664,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"1145.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512x4x4</text>\n</g>\n<!-- 9442230734797139557&#45;&gt;/outputs/63 -->\n<g id=\"edge21\" class=\"edge\">\n<title>9442230734797139557&#45;&gt;/outputs/63</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1433.3183,-26C1457.2728,-26 1487.2334,-26 1511.6669,-26\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1511.7664,-29.5001 1521.7663,-26 1511.7663,-22.5001 1511.7664,-29.5001\"/>\n<text text-anchor=\"middle\" x=\"1477.5\" y=\"-29\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">12x512x2x2</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RAodboScj5w6"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Dzh3ROmRj5w7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a680f45b-5fcc-41f3-e3db-e5a45d406d62"
      },
      "source": [
        "def compute_accuracy(model, data_loader):\n",
        "    model.eval()\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for i, (features, targets) in enumerate(data_loader):\n",
        "            \n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "\n",
        "\n",
        "def compute_epoch_loss(model, data_loader):\n",
        "    model.eval()\n",
        "    curr_loss, num_examples = 0., 0\n",
        "    with torch.no_grad():\n",
        "        for features, targets in data_loader:\n",
        "            features = features.to(DEVICE)\n",
        "            targets = targets.to(DEVICE)\n",
        "            logits, probas = model(features)\n",
        "            loss = F.cross_entropy(logits, targets, reduction='sum')\n",
        "            num_examples += targets.size(0)\n",
        "            curr_loss += loss\n",
        "\n",
        "        curr_loss = curr_loss / num_examples\n",
        "        return curr_loss\n",
        "    \n",
        "    \n",
        "\n",
        "start_time = time.time()\n",
        "for epoch in range(num_epochs):\n",
        "    \n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 50:\n",
        "            print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
        "                   %(epoch+1, num_epochs, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False): # save memory during inference\n",
        "        print('Epoch: %03d/%03d | Train: %.3f%% |  Loss: %.3f' % (\n",
        "              epoch+1, num_epochs, \n",
        "              compute_accuracy(model, train_loader),\n",
        "              compute_epoch_loss(model, train_loader)))\n",
        "\n",
        "\n",
        "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
        "    \n",
        "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/010 | Batch 0000/0391 | Cost: 2.4443\n",
            "Epoch: 001/010 | Batch 0050/0391 | Cost: 2.1140\n",
            "Epoch: 001/010 | Batch 0100/0391 | Cost: 2.0593\n",
            "Epoch: 001/010 | Batch 0150/0391 | Cost: 1.9448\n",
            "Epoch: 001/010 | Batch 0200/0391 | Cost: 2.0826\n",
            "Epoch: 001/010 | Batch 0250/0391 | Cost: 1.8528\n",
            "Epoch: 001/010 | Batch 0300/0391 | Cost: 1.7034\n",
            "Epoch: 001/010 | Batch 0350/0391 | Cost: 1.7008\n",
            "Epoch: 001/010 | Train: 32.768% |  Loss: 1.771\n",
            "Time elapsed: 2.25 min\n",
            "Epoch: 002/010 | Batch 0000/0391 | Cost: 1.7777\n",
            "Epoch: 002/010 | Batch 0050/0391 | Cost: 1.6663\n",
            "Epoch: 002/010 | Batch 0100/0391 | Cost: 1.5214\n",
            "Epoch: 002/010 | Batch 0150/0391 | Cost: 1.5125\n",
            "Epoch: 002/010 | Batch 0200/0391 | Cost: 1.3399\n",
            "Epoch: 002/010 | Batch 0250/0391 | Cost: 1.3863\n",
            "Epoch: 002/010 | Batch 0300/0391 | Cost: 1.4996\n",
            "Epoch: 002/010 | Batch 0350/0391 | Cost: 1.2441\n",
            "Epoch: 002/010 | Train: 51.818% |  Loss: 1.309\n",
            "Time elapsed: 4.53 min\n",
            "Epoch: 003/010 | Batch 0000/0391 | Cost: 1.2667\n",
            "Epoch: 003/010 | Batch 0050/0391 | Cost: 1.2038\n",
            "Epoch: 003/010 | Batch 0100/0391 | Cost: 1.2838\n",
            "Epoch: 003/010 | Batch 0150/0391 | Cost: 1.1341\n",
            "Epoch: 003/010 | Batch 0200/0391 | Cost: 1.2569\n",
            "Epoch: 003/010 | Batch 0250/0391 | Cost: 0.8705\n",
            "Epoch: 003/010 | Batch 0300/0391 | Cost: 1.1947\n",
            "Epoch: 003/010 | Batch 0350/0391 | Cost: 1.0217\n",
            "Epoch: 003/010 | Train: 61.460% |  Loss: 1.077\n",
            "Time elapsed: 6.81 min\n",
            "Epoch: 004/010 | Batch 0000/0391 | Cost: 1.0477\n",
            "Epoch: 004/010 | Batch 0050/0391 | Cost: 0.9024\n",
            "Epoch: 004/010 | Batch 0100/0391 | Cost: 1.0020\n",
            "Epoch: 004/010 | Batch 0150/0391 | Cost: 0.8484\n",
            "Epoch: 004/010 | Batch 0200/0391 | Cost: 0.9994\n",
            "Epoch: 004/010 | Batch 0250/0391 | Cost: 1.0136\n",
            "Epoch: 004/010 | Batch 0300/0391 | Cost: 0.9009\n",
            "Epoch: 004/010 | Batch 0350/0391 | Cost: 0.8051\n",
            "Epoch: 004/010 | Train: 71.906% |  Loss: 0.822\n",
            "Time elapsed: 9.10 min\n",
            "Epoch: 005/010 | Batch 0000/0391 | Cost: 0.7654\n",
            "Epoch: 005/010 | Batch 0050/0391 | Cost: 0.7165\n",
            "Epoch: 005/010 | Batch 0100/0391 | Cost: 0.8538\n",
            "Epoch: 005/010 | Batch 0150/0391 | Cost: 0.7705\n",
            "Epoch: 005/010 | Batch 0200/0391 | Cost: 0.7445\n",
            "Epoch: 005/010 | Batch 0250/0391 | Cost: 0.6507\n",
            "Epoch: 005/010 | Batch 0300/0391 | Cost: 0.6031\n",
            "Epoch: 005/010 | Batch 0350/0391 | Cost: 0.6900\n",
            "Epoch: 005/010 | Train: 75.538% |  Loss: 0.695\n",
            "Time elapsed: 11.38 min\n",
            "Epoch: 006/010 | Batch 0000/0391 | Cost: 0.6272\n",
            "Epoch: 006/010 | Batch 0050/0391 | Cost: 0.6305\n",
            "Epoch: 006/010 | Batch 0100/0391 | Cost: 0.6741\n",
            "Epoch: 006/010 | Batch 0150/0391 | Cost: 0.5443\n",
            "Epoch: 006/010 | Batch 0200/0391 | Cost: 0.5885\n",
            "Epoch: 006/010 | Batch 0250/0391 | Cost: 0.6764\n",
            "Epoch: 006/010 | Batch 0300/0391 | Cost: 0.8764\n",
            "Epoch: 006/010 | Batch 0350/0391 | Cost: 0.6619\n",
            "Epoch: 006/010 | Train: 83.040% |  Loss: 0.511\n",
            "Time elapsed: 13.67 min\n",
            "Epoch: 007/010 | Batch 0000/0391 | Cost: 0.5779\n",
            "Epoch: 007/010 | Batch 0050/0391 | Cost: 0.5978\n",
            "Epoch: 007/010 | Batch 0100/0391 | Cost: 0.6729\n",
            "Epoch: 007/010 | Batch 0150/0391 | Cost: 0.6306\n",
            "Epoch: 007/010 | Batch 0200/0391 | Cost: 0.5560\n",
            "Epoch: 007/010 | Batch 0250/0391 | Cost: 0.7424\n",
            "Epoch: 007/010 | Batch 0300/0391 | Cost: 0.7919\n",
            "Epoch: 007/010 | Batch 0350/0391 | Cost: 0.6517\n",
            "Epoch: 007/010 | Train: 81.884% |  Loss: 0.541\n",
            "Time elapsed: 15.95 min\n",
            "Epoch: 008/010 | Batch 0000/0391 | Cost: 0.4989\n",
            "Epoch: 008/010 | Batch 0050/0391 | Cost: 0.4674\n",
            "Epoch: 008/010 | Batch 0100/0391 | Cost: 0.4771\n",
            "Epoch: 008/010 | Batch 0150/0391 | Cost: 0.4234\n",
            "Epoch: 008/010 | Batch 0200/0391 | Cost: 0.6247\n",
            "Epoch: 008/010 | Batch 0250/0391 | Cost: 0.5857\n",
            "Epoch: 008/010 | Batch 0300/0391 | Cost: 0.6818\n",
            "Epoch: 008/010 | Batch 0350/0391 | Cost: 0.3931\n",
            "Epoch: 008/010 | Train: 85.708% |  Loss: 0.433\n",
            "Time elapsed: 18.24 min\n",
            "Epoch: 009/010 | Batch 0000/0391 | Cost: 0.4620\n",
            "Epoch: 009/010 | Batch 0050/0391 | Cost: 0.4814\n",
            "Epoch: 009/010 | Batch 0100/0391 | Cost: 0.4274\n",
            "Epoch: 009/010 | Batch 0150/0391 | Cost: 0.6495\n",
            "Epoch: 009/010 | Batch 0200/0391 | Cost: 0.3960\n",
            "Epoch: 009/010 | Batch 0250/0391 | Cost: 0.4977\n",
            "Epoch: 009/010 | Batch 0300/0391 | Cost: 0.4766\n",
            "Epoch: 009/010 | Batch 0350/0391 | Cost: 0.5426\n",
            "Epoch: 009/010 | Train: 88.704% |  Loss: 0.340\n",
            "Time elapsed: 20.52 min\n",
            "Epoch: 010/010 | Batch 0000/0391 | Cost: 0.2259\n",
            "Epoch: 010/010 | Batch 0050/0391 | Cost: 0.3996\n",
            "Epoch: 010/010 | Batch 0100/0391 | Cost: 0.3136\n",
            "Epoch: 010/010 | Batch 0150/0391 | Cost: 0.3979\n",
            "Epoch: 010/010 | Batch 0200/0391 | Cost: 0.4190\n",
            "Epoch: 010/010 | Batch 0250/0391 | Cost: 0.3493\n",
            "Epoch: 010/010 | Batch 0300/0391 | Cost: 0.4644\n",
            "Epoch: 010/010 | Batch 0350/0391 | Cost: 0.5054\n",
            "Epoch: 010/010 | Train: 88.994% |  Loss: 0.328\n",
            "Time elapsed: 22.79 min\n",
            "Total Training Time: 22.79 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "paaeEQHQj5xC"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gzQMWKq5j5xE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f8165adf-8009-464a-e2f8-a946a5d8f150"
      },
      "source": [
        "with torch.set_grad_enabled(False): # save memory during inference\n",
        "    print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 76.00%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRfP-97neo2E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "d63ff23d-21a5-4890-f4b5-6ce19c52e8e7"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch 1.5.1+cu101\n",
            "numpy 1.18.5\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}