PyTorch_Dropout.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PyTorch Dropout.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/WybeKoper/Dropout-reproduction/blob/master/PyTorch_Dropout.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tBnbqGFR_9hn",
        "colab_type": "text"
      },
      "source": [
        "# Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77gAc5_6AAzw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch as th\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import torchvision as tv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q1HP02lALbpi",
        "colab_type": "text"
      },
      "source": [
        "# Dataset Information Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErbjOhI4LWNe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DataSetInfo(object):\n",
        "    r\"\"\"DataSet Information Class \n",
        "\n",
        "    Args:\n",
        "        name:               name of distribution.\n",
        "        dataset:            parameter of the distribution.\n",
        "        transforms:         All transformations to be applied to the dataset.\n",
        "\n",
        "    \"\"\"\n",
        "    def __init__(self, name, dataset, transform):\n",
        "        self.name = name\n",
        "        self.dataset = dataset\n",
        "        self.transform = transform"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OOpKys-7MNO3",
        "colab_type": "text"
      },
      "source": [
        "# Dataset Initializer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntr9N4M-A-x2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DataSet(object):\n",
        "    r\"\"\"Class DataSet downloads dataset from torchvision and \n",
        "    initilizer the training and testing batch loaders.\n",
        "\n",
        "    Args:\n",
        "        name:               name of distribution.\n",
        "        dataset:            parameter of the distribution.\n",
        "        transform:         All transformations to be applied to the dataset.\n",
        "        train_batch_size:   Size of the training data batch size.\n",
        "        test_batch_size:    Size of the testing data batch size.\n",
        "        directory:          Target directory location of the data.\n",
        "\n",
        "    \"\"\"\n",
        "    def __init__(self, name, dataset, transform, train_batch_size, test_batch_size, directory='./data'):\n",
        "        self.name = name\n",
        "        \n",
        "        self.train_dataset = dataset(root=directory,\n",
        "                                    train=True,\n",
        "                                    transform=transform,\n",
        "                                    download=True)\n",
        "\n",
        "        self.train_loader = th.utils.data.DataLoader(dataset=self.train_dataset,\n",
        "                                    batch_size=train_batch_size,\n",
        "                                    shuffle=True)\n",
        "\n",
        "        self.test_dataset = dataset(root=directory,\n",
        "                                    train=False,\n",
        "                                    transform=transform,\n",
        "                                    download=True)\n",
        "\n",
        "        self.test_loader = th.utils.data.DataLoader(dataset=self.test_dataset,\n",
        "                                    batch_size=test_batch_size,\n",
        "                                    shuffle=False)\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mq645lHEQo5C",
        "colab_type": "text"
      },
      "source": [
        "# Bernoulli and Gaussian Dropout Class\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiEp1iCSQbBq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class GaussianDropout(th.nn.Module):\n",
        "    r\"\"\"During training, randomly multiplies some of the elements of the input\n",
        "    tensor with sample noise of the Gaussian distribution e ~ N(1, variance).\n",
        "    Expected value of the random variable is 1.\n",
        "    Variance of the random variable is (1 - p) / p.\n",
        "\n",
        "    Args:\n",
        "        variance: name of distribution\n",
        "\n",
        "    p is the retention rate in the case of the Bernoulli distribution\n",
        "    p defines the variance in the case of the Gaussian distribution\n",
        "    \"\"\"\n",
        "    def __init__(self, variance=1.0):\n",
        "        super(GaussianDropout, self).__init__()\n",
        "        self.variance = th.Tensor([variance])\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Sample noise   e ~ N(1, variance)\n",
        "        Multiply noise h = h_ * e\n",
        "        \"\"\"\n",
        "        if self.train():\n",
        "            epsilon = th.autograd.Variable(1. + th.randn(x.size()) * self.variance)\n",
        "            if x.is_cuda:\n",
        "                epsilon = epsilon.cuda()\n",
        "            return x * epsilon\n",
        "        else:\n",
        "            return x\n",
        "\n",
        "\n",
        "def dropout(method, q):\n",
        "    r\"\"\"During training, randomly zeroes some of the elements of the input\n",
        "    tensor with probability p. q is retention rate and equals q = 1 - p.\n",
        "    Expected value of the random variable is 1.\n",
        "    Variance of the random variable is (1 - p) / p.\n",
        "\n",
        "    Args:\n",
        "        method: name of distribution\n",
        "        p: parameter of the distribution.\n",
        "\n",
        "    p is the retention rate in the case of the Bernoulli distribution\n",
        "    p defines the variance in the case of the Gaussian distribution\n",
        "    \"\"\"\n",
        "    if method == 'Bernoulli':\n",
        "        return th.nn.Dropout(p=(1-q))\n",
        "    elif method == 'Gaussian':\n",
        "        return GaussianDropout(variance=((1 - q) / q))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n8l_tmjgU3p_",
        "colab_type": "text"
      },
      "source": [
        "# Convolutional Neural Networks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_K551WWaU2w7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MNISTNet(th.nn.Module):\n",
        "    r\"\"\"Neural Network for MNIST dataset\n",
        "\n",
        "    Args:\n",
        "        dropout_method: name of distribution used for dropout\n",
        "\n",
        "    q is the retention rate in the case of the Bernoulli distribution\n",
        "    Gaussian parameter is calculated as: (1 - q) / q\n",
        "    \"\"\"\n",
        "    def __init__(self, dropout_method, batch_size):\n",
        "        super(MNISTNet, self).__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.input_drop = dropout(method=dropout_method, q=0.8)\n",
        "        self.hidden1 = th.nn.Linear(28 * 28, 1024)\n",
        "        self.hidden1_drop = dropout(method=dropout_method, q=0.5)\n",
        "        self.hidden2 = th.nn.Linear(1024, 1024)\n",
        "        self.hidden2_drop = dropout(method=dropout_method, q=0.5)\n",
        "        self.output = th.nn.Linear(1024, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x.view(self.batch_size, -1)\n",
        "        x = self.input_drop(x)\n",
        "        x = F.relu(self.hidden1_drop(self.hidden1(x)))\n",
        "        x = F.relu(self.hidden2_drop(self.hidden2(x)))\n",
        "        x = self.output(x)\n",
        "        \n",
        "        return F.log_softmax(x, dim=1)\n",
        "\n",
        "class CIFAR10Net(th.nn.Module):\n",
        "    r\"\"\"Neural Network for CIFAR-10 dataset\n",
        "\n",
        "    Args:\n",
        "        dropout_method: name of distribution used for dropout\n",
        "        p: parameter of the distribution.\n",
        "\n",
        "    q is the retention rate in the case of the Bernoulli distribution\n",
        "    Gaussian parameter is calculated as: (1 - q) / q\n",
        "    \"\"\"\n",
        "    def __init__(self, dropout_method, batch_size):\n",
        "        super(CIFAR10Net, self).__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.input_drop = dropout(method=dropout_method, q=0.9)\n",
        "\n",
        "        self.conv1 = th.nn.Conv2d(in_channels=3, out_channels=96, kernel_size=5, stride=1, padding=2)\n",
        "        self.conv1_drop = dropout(method=dropout_method, q=0.75)\n",
        "        self.conv1_pool = th.nn.MaxPool2d(kernel_size=3, stride=2)\n",
        "\n",
        "        self.conv2 = th.nn.Conv2d(in_channels=96, out_channels=128, kernel_size=5, stride=1, padding=2)\n",
        "        self.conv2_drop = dropout(method=dropout_method, q=0.75)\n",
        "        self.conv2_pool = th.nn.MaxPool2d(kernel_size=3, stride=2)\n",
        "\n",
        "        self.conv3 = th.nn.Conv2d(in_channels=128, out_channels=256, kernel_size=5, stride=1, padding=2)\n",
        "        self.conv3_drop = dropout(method=dropout_method, q=0.5)\n",
        "        self.conv3_pool = th.nn.MaxPool2d(kernel_size=3, stride=2)\n",
        "\n",
        "        self.fc1 = th.nn.Linear(2304, 2048)\n",
        "        self.fc1_drop = dropout(method=dropout_method, q=0.5)\n",
        "\n",
        "        self.fc2 = th.nn.Linear(2048, 2048)\n",
        "        self.fc2_drop = dropout(method=dropout_method, q=0.5)\n",
        "\n",
        "        self.output = th.nn.Linear(2048, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.input_drop(x)\n",
        "        x = self.conv1_pool(self.conv1_drop(self.conv1(x)))\n",
        "        x = self.conv2_pool(self.conv2_drop(self.conv2(x)))\n",
        "        x = self.conv3_pool(self.conv3_drop(self.conv3(x)))\n",
        "        x = x.view(self.batch_size, -1)\n",
        "        x = F.relu(self.fc1_drop(self.fc1(x)))\n",
        "        x = F.relu(self.fc2_drop(self.fc2(x)))\n",
        "        x = self.output(x)\n",
        "\n",
        "        return F.log_softmax(x, dim=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iIZuj3GjBhtq",
        "colab_type": "text"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YskgnkMzPVLc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model(object):\n",
        "    r\"\"\"Model for problem\n",
        "\n",
        "    Args:\n",
        "        ds:                 dataset object with name, location and transforms, accept MNIST and CIFAR-10\n",
        "        dropout_method:     name of distribution used for dropout, accepts Bernoulli or Gaussian\n",
        "        hp:                 hyperparameters, requires epochs, train_batch_size, rest_batch_size and lr\n",
        "        criterion:          Loss Function\n",
        "\n",
        "    q is the retention rate in the case of the Bernoulli distribution\n",
        "    Gaussian parameter is calculated as: (1 - q) / q\n",
        "    \"\"\"\n",
        "    def __init__(self, ds, dropout_method, hp, criterion):\n",
        "        self.ds = ds\n",
        "        if self.ds.name not in ['MNIST', 'CIFAR-10']:\n",
        "            print('Your dataset is not accepted, must be \"MNIST\" or \"CIFAR-10\" was {}'.format(ds.name))\n",
        "\n",
        "        else:\n",
        "            self.hp = hp\n",
        "            self.dataset = DataSet(self.ds.name, self.ds.dataset, self.ds.transform, self.hp['train_batch_size'], self.hp['test_batch_size'])\n",
        "            self.train_loader = self.dataset.train_loader\n",
        "            self.test_loader = self.dataset.test_loader\n",
        "\n",
        "            self.dropout_method = dropout_method\n",
        "\n",
        "            if self.ds.name == 'MNIST':\n",
        "                self.net = MNISTNet(dropout_method=self.dropout_method, batch_size=self.hp['train_batch_size'])\n",
        "            elif self.ds.name == 'CIFAR-10':\n",
        "                self.net = CIFAR10Net(dropout_method=self.dropout_method, batch_size=self.hp['train_batch_size'])\n",
        "            \n",
        "            if self.hp['cuda']:\n",
        "                self.net.cuda()\n",
        "\n",
        "            self.optimizer = None\n",
        "\n",
        "            self.criterion = criterion\n",
        "\n",
        "    def train(self, epoch):\n",
        "        self.net.train()\n",
        "        for batch_idx, (data, target) in enumerate(self.train_loader):\n",
        "            # Change to GPU\n",
        "            if self.hp['cuda']:\n",
        "                data = data.cuda()\n",
        "                target = target.cuda()\n",
        "            #Variables in Pytorch are differenciable. \n",
        "            data = th.autograd.Variable(data)\n",
        "            target = th.autograd.Variable(target)\n",
        "            #This will zero out the gradients for this batch. \n",
        "            self.optimizer.zero_grad()\n",
        "\n",
        "            # Calculate output of Neural Net\n",
        "            output = self.net(data)\n",
        "            # Calculate the loss The Cross Entropy Loss. It is useful to train a classification problem with C classes.\n",
        "            loss = self.criterion(output, target)\n",
        "            #dloss/dx for every Variable \n",
        "            loss.backward()\n",
        "            #to do a one-step update on our parameter.\n",
        "            self.optimizer.step()\n",
        "            #Print out the loss periodically. \n",
        "            if batch_idx % self.hp['log_interval'] == 0:\n",
        "                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                    epoch, batch_idx * len(data), len(self.train_loader.dataset),\n",
        "                    100. * batch_idx / len(self.train_loader), loss.data))\n",
        "\n",
        "    def test(self):\n",
        "        self.net.eval()\n",
        "        test_loss = 0\n",
        "        correct = 0\n",
        "        for data, target in self.test_loader:\n",
        "            #Change to GPU\n",
        "            if self.hp['cuda']:\n",
        "                data = data.cuda()\n",
        "                target = target.cuda()\n",
        "            #Variables in Pytorch are differenciable. \n",
        "            data = th.autograd.Variable(data)\n",
        "            target = th.autograd.Variable(target)\n",
        "            \n",
        "            # Calculate output of Neural Net\n",
        "            output = self.net(data)\n",
        "            # Calculate the loss The Cross Entropy Loss. It is useful to train a classification problem with C classes.\n",
        "            test_loss += self.criterion(output, target)\n",
        "            # get the index of the max log-probability\n",
        "            pred = output.data.max(1, keepdim=True)[1] \n",
        "            correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()\n",
        "\n",
        "        test_loss /= len(self.test_loader.dataset)\n",
        "        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.1f}%), Error Rate {:.1f}\\n'.format(\n",
        "            test_loss, \n",
        "            correct, \n",
        "            len(self.test_loader.dataset),\n",
        "            100. * correct / len(self.test_loader.dataset), \n",
        "            100. * (len(self.test_loader.dataset)-correct) / len(self.test_loader.dataset)\n",
        "            ))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BvSxLxo6EOHR",
        "colab_type": "text"
      },
      "source": [
        "# Solver"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XejM7yrhEQl-",
        "colab_type": "code",
        "outputId": "52b08f23-88c8-4d0b-b991-966872b95614",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "hyperparameters = {\n",
        "    'train_batch_size': 500,\n",
        "    'test_batch_size': 500,\n",
        "    'epochs': 50,\n",
        "    'lr': 0.001,\n",
        "    'log_interval': 10,\n",
        "    'cuda': True,\n",
        "    'seed': 42,                 # Not used atm\n",
        "    'momentum': 0.5             # Not used atm\n",
        "}\n",
        "\n",
        "mnist = DataSetInfo('MNIST', \n",
        "                    tv.datasets.MNIST, \n",
        "                    tv.transforms.ToTensor())\n",
        "\n",
        "cifar10 = DataSetInfo('CIFAR-10', \n",
        "                      tv.datasets.CIFAR10, \n",
        "                      tv.transforms.Compose([\n",
        "                                             tv.transforms.ToTensor(), \n",
        "                                             tv.transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]))\n",
        "\n",
        "criterion = th.nn.CrossEntropyLoss()\n",
        "\n",
        "model = Model(mnist, 'Gaussian', hyperparameters, criterion)\n",
        "\n",
        "model.optimizer = th.optim.Adam(model.net.parameters(), lr=hyperparameters['lr'])\n",
        "\n",
        "for epoch in range(1, hyperparameters['epochs'] + 1):\n",
        "    model.train(epoch)\n",
        "    model.test()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.303205\n",
            "Train Epoch: 1 [5000/60000 (8%)]\tLoss: 0.711732\n",
            "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.482109\n",
            "Train Epoch: 1 [15000/60000 (25%)]\tLoss: 0.392214\n",
            "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.376645\n",
            "Train Epoch: 1 [25000/60000 (42%)]\tLoss: 0.311821\n",
            "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.296066\n",
            "Train Epoch: 1 [35000/60000 (58%)]\tLoss: 0.272529\n",
            "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.211156\n",
            "Train Epoch: 1 [45000/60000 (75%)]\tLoss: 0.202527\n",
            "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.246218\n",
            "Train Epoch: 1 [55000/60000 (92%)]\tLoss: 0.163795\n",
            "\n",
            "Test set: Average loss: 0.0003, Accuracy: 9505/10000 (95.1%) error rate 4.9\n",
            "\n",
            "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.128424\n",
            "Train Epoch: 2 [5000/60000 (8%)]\tLoss: 0.182384\n",
            "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.190731\n",
            "Train Epoch: 2 [15000/60000 (25%)]\tLoss: 0.150395\n",
            "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.124142\n",
            "Train Epoch: 2 [25000/60000 (42%)]\tLoss: 0.185008\n",
            "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.148389\n",
            "Train Epoch: 2 [35000/60000 (58%)]\tLoss: 0.175068\n",
            "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.138795\n",
            "Train Epoch: 2 [45000/60000 (75%)]\tLoss: 0.205183\n",
            "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.157639\n",
            "Train Epoch: 2 [55000/60000 (92%)]\tLoss: 0.196402\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9635/10000 (96.3%) error rate 3.7\n",
            "\n",
            "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.148211\n",
            "Train Epoch: 3 [5000/60000 (8%)]\tLoss: 0.074226\n",
            "Train Epoch: 3 [10000/60000 (17%)]\tLoss: 0.101276\n",
            "Train Epoch: 3 [15000/60000 (25%)]\tLoss: 0.091438\n",
            "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 0.106389\n",
            "Train Epoch: 3 [25000/60000 (42%)]\tLoss: 0.107769\n",
            "Train Epoch: 3 [30000/60000 (50%)]\tLoss: 0.101850\n",
            "Train Epoch: 3 [35000/60000 (58%)]\tLoss: 0.135458\n",
            "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 0.126166\n",
            "Train Epoch: 3 [45000/60000 (75%)]\tLoss: 0.095240\n",
            "Train Epoch: 3 [50000/60000 (83%)]\tLoss: 0.120535\n",
            "Train Epoch: 3 [55000/60000 (92%)]\tLoss: 0.089242\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9645/10000 (96.4%) error rate 3.5\n",
            "\n",
            "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.096755\n",
            "Train Epoch: 4 [5000/60000 (8%)]\tLoss: 0.079677\n",
            "Train Epoch: 4 [10000/60000 (17%)]\tLoss: 0.075038\n",
            "Train Epoch: 4 [15000/60000 (25%)]\tLoss: 0.069805\n",
            "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 0.050202\n",
            "Train Epoch: 4 [25000/60000 (42%)]\tLoss: 0.086075\n",
            "Train Epoch: 4 [30000/60000 (50%)]\tLoss: 0.063223\n",
            "Train Epoch: 4 [35000/60000 (58%)]\tLoss: 0.083353\n",
            "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 0.088185\n",
            "Train Epoch: 4 [45000/60000 (75%)]\tLoss: 0.069697\n",
            "Train Epoch: 4 [50000/60000 (83%)]\tLoss: 0.051188\n",
            "Train Epoch: 4 [55000/60000 (92%)]\tLoss: 0.070911\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9696/10000 (97.0%) error rate 3.0\n",
            "\n",
            "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.107451\n",
            "Train Epoch: 5 [5000/60000 (8%)]\tLoss: 0.091399\n",
            "Train Epoch: 5 [10000/60000 (17%)]\tLoss: 0.061901\n",
            "Train Epoch: 5 [15000/60000 (25%)]\tLoss: 0.092100\n",
            "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 0.075467\n",
            "Train Epoch: 5 [25000/60000 (42%)]\tLoss: 0.115761\n",
            "Train Epoch: 5 [30000/60000 (50%)]\tLoss: 0.080038\n",
            "Train Epoch: 5 [35000/60000 (58%)]\tLoss: 0.055815\n",
            "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 0.112116\n",
            "Train Epoch: 5 [45000/60000 (75%)]\tLoss: 0.078737\n",
            "Train Epoch: 5 [50000/60000 (83%)]\tLoss: 0.105008\n",
            "Train Epoch: 5 [55000/60000 (92%)]\tLoss: 0.073078\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9691/10000 (96.9%) error rate 3.1\n",
            "\n",
            "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.077563\n",
            "Train Epoch: 6 [5000/60000 (8%)]\tLoss: 0.065902\n",
            "Train Epoch: 6 [10000/60000 (17%)]\tLoss: 0.058833\n",
            "Train Epoch: 6 [15000/60000 (25%)]\tLoss: 0.092876\n",
            "Train Epoch: 6 [20000/60000 (33%)]\tLoss: 0.066351\n",
            "Train Epoch: 6 [25000/60000 (42%)]\tLoss: 0.039765\n",
            "Train Epoch: 6 [30000/60000 (50%)]\tLoss: 0.100000\n",
            "Train Epoch: 6 [35000/60000 (58%)]\tLoss: 0.077571\n",
            "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 0.081072\n",
            "Train Epoch: 6 [45000/60000 (75%)]\tLoss: 0.072118\n",
            "Train Epoch: 6 [50000/60000 (83%)]\tLoss: 0.094207\n",
            "Train Epoch: 6 [55000/60000 (92%)]\tLoss: 0.066161\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9704/10000 (97.0%) error rate 3.0\n",
            "\n",
            "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.060526\n",
            "Train Epoch: 7 [5000/60000 (8%)]\tLoss: 0.043427\n",
            "Train Epoch: 7 [10000/60000 (17%)]\tLoss: 0.087653\n",
            "Train Epoch: 7 [15000/60000 (25%)]\tLoss: 0.046312\n",
            "Train Epoch: 7 [20000/60000 (33%)]\tLoss: 0.066404\n",
            "Train Epoch: 7 [25000/60000 (42%)]\tLoss: 0.037789\n",
            "Train Epoch: 7 [30000/60000 (50%)]\tLoss: 0.058809\n",
            "Train Epoch: 7 [35000/60000 (58%)]\tLoss: 0.077011\n",
            "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 0.071612\n",
            "Train Epoch: 7 [45000/60000 (75%)]\tLoss: 0.057556\n",
            "Train Epoch: 7 [50000/60000 (83%)]\tLoss: 0.068248\n",
            "Train Epoch: 7 [55000/60000 (92%)]\tLoss: 0.081596\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9732/10000 (97.3%) error rate 2.7\n",
            "\n",
            "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.049865\n",
            "Train Epoch: 8 [5000/60000 (8%)]\tLoss: 0.065469\n",
            "Train Epoch: 8 [10000/60000 (17%)]\tLoss: 0.032098\n",
            "Train Epoch: 8 [15000/60000 (25%)]\tLoss: 0.067323\n",
            "Train Epoch: 8 [20000/60000 (33%)]\tLoss: 0.060366\n",
            "Train Epoch: 8 [25000/60000 (42%)]\tLoss: 0.036856\n",
            "Train Epoch: 8 [30000/60000 (50%)]\tLoss: 0.046398\n",
            "Train Epoch: 8 [35000/60000 (58%)]\tLoss: 0.074681\n",
            "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 0.088356\n",
            "Train Epoch: 8 [45000/60000 (75%)]\tLoss: 0.082032\n",
            "Train Epoch: 8 [50000/60000 (83%)]\tLoss: 0.050210\n",
            "Train Epoch: 8 [55000/60000 (92%)]\tLoss: 0.061378\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9712/10000 (97.1%) error rate 2.9\n",
            "\n",
            "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.062506\n",
            "Train Epoch: 9 [5000/60000 (8%)]\tLoss: 0.044783\n",
            "Train Epoch: 9 [10000/60000 (17%)]\tLoss: 0.038565\n",
            "Train Epoch: 9 [15000/60000 (25%)]\tLoss: 0.047764\n",
            "Train Epoch: 9 [20000/60000 (33%)]\tLoss: 0.059464\n",
            "Train Epoch: 9 [25000/60000 (42%)]\tLoss: 0.051568\n",
            "Train Epoch: 9 [30000/60000 (50%)]\tLoss: 0.058455\n",
            "Train Epoch: 9 [35000/60000 (58%)]\tLoss: 0.041115\n",
            "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 0.083121\n",
            "Train Epoch: 9 [45000/60000 (75%)]\tLoss: 0.076869\n",
            "Train Epoch: 9 [50000/60000 (83%)]\tLoss: 0.033583\n",
            "Train Epoch: 9 [55000/60000 (92%)]\tLoss: 0.049226\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9693/10000 (96.9%) error rate 3.1\n",
            "\n",
            "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.090142\n",
            "Train Epoch: 10 [5000/60000 (8%)]\tLoss: 0.029859\n",
            "Train Epoch: 10 [10000/60000 (17%)]\tLoss: 0.064527\n",
            "Train Epoch: 10 [15000/60000 (25%)]\tLoss: 0.027154\n",
            "Train Epoch: 10 [20000/60000 (33%)]\tLoss: 0.068896\n",
            "Train Epoch: 10 [25000/60000 (42%)]\tLoss: 0.053704\n",
            "Train Epoch: 10 [30000/60000 (50%)]\tLoss: 0.084513\n",
            "Train Epoch: 10 [35000/60000 (58%)]\tLoss: 0.033612\n",
            "Train Epoch: 10 [40000/60000 (67%)]\tLoss: 0.043777\n",
            "Train Epoch: 10 [45000/60000 (75%)]\tLoss: 0.031612\n",
            "Train Epoch: 10 [50000/60000 (83%)]\tLoss: 0.032051\n",
            "Train Epoch: 10 [55000/60000 (92%)]\tLoss: 0.041044\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9758/10000 (97.6%) error rate 2.4\n",
            "\n",
            "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.015168\n",
            "Train Epoch: 11 [5000/60000 (8%)]\tLoss: 0.049247\n",
            "Train Epoch: 11 [10000/60000 (17%)]\tLoss: 0.054743\n",
            "Train Epoch: 11 [15000/60000 (25%)]\tLoss: 0.034946\n",
            "Train Epoch: 11 [20000/60000 (33%)]\tLoss: 0.025429\n",
            "Train Epoch: 11 [25000/60000 (42%)]\tLoss: 0.037543\n",
            "Train Epoch: 11 [30000/60000 (50%)]\tLoss: 0.046620\n",
            "Train Epoch: 11 [35000/60000 (58%)]\tLoss: 0.045651\n",
            "Train Epoch: 11 [40000/60000 (67%)]\tLoss: 0.038272\n",
            "Train Epoch: 11 [45000/60000 (75%)]\tLoss: 0.081321\n",
            "Train Epoch: 11 [50000/60000 (83%)]\tLoss: 0.082447\n",
            "Train Epoch: 11 [55000/60000 (92%)]\tLoss: 0.051067\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9728/10000 (97.3%) error rate 2.7\n",
            "\n",
            "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.058973\n",
            "Train Epoch: 12 [5000/60000 (8%)]\tLoss: 0.037903\n",
            "Train Epoch: 12 [10000/60000 (17%)]\tLoss: 0.029726\n",
            "Train Epoch: 12 [15000/60000 (25%)]\tLoss: 0.028803\n",
            "Train Epoch: 12 [20000/60000 (33%)]\tLoss: 0.060379\n",
            "Train Epoch: 12 [25000/60000 (42%)]\tLoss: 0.023570\n",
            "Train Epoch: 12 [30000/60000 (50%)]\tLoss: 0.052322\n",
            "Train Epoch: 12 [35000/60000 (58%)]\tLoss: 0.055518\n",
            "Train Epoch: 12 [40000/60000 (67%)]\tLoss: 0.053676\n",
            "Train Epoch: 12 [45000/60000 (75%)]\tLoss: 0.031993\n",
            "Train Epoch: 12 [50000/60000 (83%)]\tLoss: 0.027907\n",
            "Train Epoch: 12 [55000/60000 (92%)]\tLoss: 0.019744\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9702/10000 (97.0%) error rate 3.0\n",
            "\n",
            "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.052700\n",
            "Train Epoch: 13 [5000/60000 (8%)]\tLoss: 0.035165\n",
            "Train Epoch: 13 [10000/60000 (17%)]\tLoss: 0.032527\n",
            "Train Epoch: 13 [15000/60000 (25%)]\tLoss: 0.057296\n",
            "Train Epoch: 13 [20000/60000 (33%)]\tLoss: 0.032897\n",
            "Train Epoch: 13 [25000/60000 (42%)]\tLoss: 0.041667\n",
            "Train Epoch: 13 [30000/60000 (50%)]\tLoss: 0.021320\n",
            "Train Epoch: 13 [35000/60000 (58%)]\tLoss: 0.049234\n",
            "Train Epoch: 13 [40000/60000 (67%)]\tLoss: 0.038769\n",
            "Train Epoch: 13 [45000/60000 (75%)]\tLoss: 0.041184\n",
            "Train Epoch: 13 [50000/60000 (83%)]\tLoss: 0.049346\n",
            "Train Epoch: 13 [55000/60000 (92%)]\tLoss: 0.053365\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9773/10000 (97.7%) error rate 2.3\n",
            "\n",
            "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.011685\n",
            "Train Epoch: 14 [5000/60000 (8%)]\tLoss: 0.047480\n",
            "Train Epoch: 14 [10000/60000 (17%)]\tLoss: 0.044092\n",
            "Train Epoch: 14 [15000/60000 (25%)]\tLoss: 0.039883\n",
            "Train Epoch: 14 [20000/60000 (33%)]\tLoss: 0.036223\n",
            "Train Epoch: 14 [25000/60000 (42%)]\tLoss: 0.047005\n",
            "Train Epoch: 14 [30000/60000 (50%)]\tLoss: 0.030981\n",
            "Train Epoch: 14 [35000/60000 (58%)]\tLoss: 0.058462\n",
            "Train Epoch: 14 [40000/60000 (67%)]\tLoss: 0.038485\n",
            "Train Epoch: 14 [45000/60000 (75%)]\tLoss: 0.015599\n",
            "Train Epoch: 14 [50000/60000 (83%)]\tLoss: 0.020447\n",
            "Train Epoch: 14 [55000/60000 (92%)]\tLoss: 0.030490\n",
            "\n",
            "Test set: Average loss: 0.0002, Accuracy: 9769/10000 (97.7%) error rate 2.3\n",
            "\n",
            "Train Epoch: 15 [0/60000 (0%)]\tLoss: 0.050569\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}