pytorch/PyTorch NN Basics - Autograd Gradient Neural Network Loss Backprop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from  __future__ import print_function\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1.00000e-19 *\n",
      "  0.0000  1.0842  0.0000  1.0842\n",
      "  0.0000  0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  0.0000\n",
      "[torch.FloatTensor of size 5x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# make a simple 5x4 matrix\n",
    "\n",
    "x = torch.Tensor(5,4)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.8279  0.2494  0.9360  0.4095\n",
      " 0.4523  0.0623  0.6327  0.8333\n",
      " 0.7154  0.2328  0.7027  0.5982\n",
      " 0.8434  0.7973  0.8574  0.6481\n",
      " 0.4558  0.9925  0.9888  0.5318\n",
      "[torch.FloatTensor of size 5x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Randdomly initialized matrix\n",
    "x = torch.rand(5,4)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 4])\n",
      "torch.Size is  however a tuple \n"
     ]
    }
   ],
   "source": [
    "print(x.size())\n",
    "print(\"torch.Size is  however a tuple \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1.3527  0.2853  1.8305  1.0669\n",
      " 1.3825  0.8608  0.7175  1.4620\n",
      " 0.9138  1.2007  1.4080  1.1309\n",
      " 1.6843  1.1873  1.5777  1.0847\n",
      " 0.4859  1.1659  1.1593  1.1505\n",
      "[torch.FloatTensor of size 5x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y = torch.rand(5, 4)\n",
    "print(x + y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1.3527  0.2853  1.8305  1.0669\n",
      " 1.3825  0.8608  0.7175  1.4620\n",
      " 0.9138  1.2007  1.4080  1.1309\n",
      " 1.6843  1.1873  1.5777  1.0847\n",
      " 0.4859  1.1659  1.1593  1.1505\n",
      "[torch.FloatTensor of size 5x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(torch.add(x, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1.3527  0.2853  1.8305  1.0669\n",
      " 1.3825  0.8608  0.7175  1.4620\n",
      " 0.9138  1.2007  1.4080  1.1309\n",
      " 1.6843  1.1873  1.5777  1.0847\n",
      " 0.4859  1.1659  1.1593  1.1505\n",
      "[torch.FloatTensor of size 5x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result = torch.Tensor(5, 4)\n",
    "torch.add(x, y, out=result)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.2494\n",
      " 0.0623\n",
      " 0.2328\n",
      " 0.7973\n",
      " 0.9925\n",
      "[torch.FloatTensor of size 5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(x[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       "[torch.FloatTensor of size 10]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting torch Tensor into a numpy Array\n",
    "a = torch.ones(10)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "b = a.numpy()\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      "[torch.FloatTensor of size 10]\n",
      "\n",
      "[ 2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]\n"
     ]
    }
   ],
   "source": [
    "# Watch the numpy array\n",
    "a.add_(1)\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]\n",
      "\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      " 2\n",
      "[torch.DoubleTensor of size 10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Doing the opposite; convertihng numpy Array into a torch Tensor\n",
    "import numpy as np\n",
    "a = np.ones(10)\n",
    "b = torch.from_numpy(a)\n",
    "np.add(a, 1, out=a)\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    x = x.cuda()\n",
    "    y = y.cuda()\n",
    "    x + y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A bit on Autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 1  1  1\n",
      " 1  1  1\n",
      " 1  1  1\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.ones(3,3), requires_grad=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 3  3  3\n",
      " 3  3  3\n",
      " 3  3  3\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "grad_fn",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-faedf5ea10b6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/torch/autograd/variable.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fallthrough_methods\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: grad_fn"
     ]
    }
   ],
   "source": [
    "print(y.grad_fn)\n",
    "#why does this fail?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 27  27  27\n",
      " 27  27  27\n",
      " 27  27  27\n",
      "[torch.FloatTensor of size 3x3]\n",
      " Variable containing:\n",
      " 27\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "z = y*y*3\n",
    "out = z.mean()\n",
    "print(z, out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gradients\n",
    "Backprop with out.backward is same as out.backward(torch.Tensor([1,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 2  2  2\n",
      " 2  2  2\n",
      " 2  2  2\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "-1358.7740\n",
      "  731.0142\n",
      "  699.5322\n",
      " -524.8378\n",
      " -568.4971\n",
      "[torch.FloatTensor of size 5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(5)\n",
    "x = Variable(x, requires_grad=True)\n",
    "y = x*2\n",
    "while y.data.norm()<1000:\n",
    "    y = y*2\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "  51.2000\n",
      " 512.0000\n",
      "   0.0512\n",
      "[torch.FloatTensor of size 3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gradients = torch.FloatTensor([0.1, 1.0, 0.0001])\n",
    "y.backward(gradients)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class Variable in module torch.autograd.variable:\n",
      "\n",
      "class Variable(torch._C._VariableBase)\n",
      " |  Wraps a tensor and records the operations applied to it.\n",
      " |  \n",
      " |  Variable is a thin wrapper around a Tensor object, that also holds\n",
      " |  the gradient w.r.t. to it, and a reference to a function that created it.\n",
      " |  This reference allows retracing the whole chain of operations that\n",
      " |  created the data. If the Variable has been created by the user, its creator\n",
      " |  will be ``None`` and we call such objects *leaf* Variables.\n",
      " |  \n",
      " |  Since autograd only supports scalar valued function differentiation, grad\n",
      " |  size always matches the data size. Also, grad is normally only allocated\n",
      " |  for leaf variables, and will be always zero otherwise.\n",
      " |  \n",
      " |  Attributes:\n",
      " |      data: Wrapped tensor of any type.\n",
      " |      grad: Variable holding the gradient of type and location matching\n",
      " |          the ``.data``.  This attribute is lazily allocated and can't\n",
      " |          be reassigned.\n",
      " |      requires_grad: Boolean indicating whether the Variable has been\n",
      " |          created by a subgraph containing any Variable, that requires it.\n",
      " |          See :ref:`excluding-subgraphs` for more details.\n",
      " |          Can be changed only on leaf Variables.\n",
      " |      volatile: Boolean indicating that the Variable should be used in\n",
      " |          inference mode, i.e. don't save the history. See\n",
      " |          :ref:`excluding-subgraphs` for more details.\n",
      " |          Can be changed only on leaf Variables.\n",
      " |      creator: Function of which the variable was an output. For leaf\n",
      " |          (user created) variables it's ``None``. Read-only attribute.\n",
      " |  \n",
      " |  Parameters:\n",
      " |      data (any tensor class): Tensor to wrap.\n",
      " |      requires_grad (bool): Value of the requires_grad flag. **Keyword only.**\n",
      " |      volatile (bool): Value of the volatile flag. **Keyword only.**\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      Variable\n",
      " |      torch._C._VariableBase\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __add__(self, other)\n",
      " |  \n",
      " |  __deepcopy__(self, memo)\n",
      " |  \n",
      " |  __div__(self, other)\n",
      " |  \n",
      " |  __eq__(self, other)\n",
      " |      Return self==value.\n",
      " |  \n",
      " |  __ge__(self, other)\n",
      " |      Return self>=value.\n",
      " |  \n",
      " |  __getattr__(self, name)\n",
      " |  \n",
      " |  __getitem__(self, key)\n",
      " |  \n",
      " |  __gt__(self, other)\n",
      " |      Return self>value.\n",
      " |  \n",
      " |  __hash__(self)\n",
      " |      Return hash(self).\n",
      " |  \n",
      " |  __iadd__(self, other)\n",
      " |  \n",
      " |  __idiv__(self, other)\n",
      " |  \n",
      " |  __imul__(self, other)\n",
      " |  \n",
      " |  __ipow__(self, other)\n",
      " |  \n",
      " |  __isub__(self, other)\n",
      " |  \n",
      " |  __iter__(self)\n",
      " |  \n",
      " |  __le__(self, other)\n",
      " |      Return self<=value.\n",
      " |  \n",
      " |  __len__(self)\n",
      " |  \n",
      " |  __lt__(self, other)\n",
      " |      Return self<value.\n",
      " |  \n",
      " |  __matmul__(self, other)\n",
      " |  \n",
      " |  __mod__(self, other)\n",
      " |  \n",
      " |  __mul__(self, other)\n",
      " |  \n",
      " |  __ne__(self, other)\n",
      " |      Return self!=value.\n",
      " |  \n",
      " |  __neg__(self)\n",
      " |  \n",
      " |  __pow__(self, other)\n",
      " |  \n",
      " |  __radd__ = __add__(self, other)\n",
      " |  \n",
      " |  __rdiv__(self, other)\n",
      " |  \n",
      " |  __reduce_ex__(self, proto)\n",
      " |      helper for pickle\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __rmul__ = __mul__(self, other)\n",
      " |  \n",
      " |  __rpow__(self, other)\n",
      " |  \n",
      " |  __rsub__(self, other)\n",
      " |  \n",
      " |  __rtruediv__ = __rdiv__(self, other)\n",
      " |  \n",
      " |  __setitem__(self, key, value)\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  __sub__(self, other)\n",
      " |  \n",
      " |  __truediv__ = __div__(self, other)\n",
      " |  \n",
      " |  abs(self)\n",
      " |  \n",
      " |  acos(self)\n",
      " |  \n",
      " |  add(self, other)\n",
      " |  \n",
      " |  add_(self, other)\n",
      " |  \n",
      " |  addbmm(self, *args)\n",
      " |  \n",
      " |  addbmm_(self, *args)\n",
      " |  \n",
      " |  addcdiv(self, *args)\n",
      " |  \n",
      " |  addcmul(self, *args)\n",
      " |  \n",
      " |  addmm(self, *args)\n",
      " |  \n",
      " |  addmm_(self, *args)\n",
      " |  \n",
      " |  addmv(self, *args)\n",
      " |  \n",
      " |  addmv_(self, *args)\n",
      " |  \n",
      " |  addr(self, *args)\n",
      " |  \n",
      " |  addr_(self, *args)\n",
      " |  \n",
      " |  asin(self)\n",
      " |  \n",
      " |  atan(self)\n",
      " |  \n",
      " |  backward(self, gradient=None, retain_variables=False)\n",
      " |      Computes the gradient of current variable w.r.t. graph leaves.\n",
      " |      \n",
      " |      The graph is differentiated using the chain rule. If the variable is\n",
      " |      non-scalar (i.e. its data has more than one element) and requires\n",
      " |      gradient, the function additionaly requires specifying ``gradient``.\n",
      " |      It should be a tensor of matching type and location, that contains\n",
      " |      the gradient of the differentiated function w.r.t. ``self``.\n",
      " |      \n",
      " |      This function accumulates gradients in the leaves - you might need to zero\n",
      " |      them before calling it.\n",
      " |      \n",
      " |      Arguments:\n",
      " |          gradient (Tensor): Gradient of the differentiated function\n",
      " |              w.r.t. the data. Required only if the data has more than one\n",
      " |              element. Type and location should match these of ``self.data``.\n",
      " |          retain_variables (bool): If ``True``, buffers necessary for computing\n",
      " |              gradients won't be freed after use. It is only necessary to\n",
      " |              specify ``True`` if you want to differentiate some subgraph multiple\n",
      " |              times (in some cases it will be much more efficient to use\n",
      " |              `autograd.backward`).\n",
      " |  \n",
      " |  baddbmm(self, *args)\n",
      " |  \n",
      " |  baddbmm_(self, *args)\n",
      " |  \n",
      " |  bernoulli(self)\n",
      " |  \n",
      " |  bmm(self, batch)\n",
      " |  \n",
      " |  byte(self)\n",
      " |  \n",
      " |  ceil(self)\n",
      " |  \n",
      " |  char(self)\n",
      " |  \n",
      " |  chunk(self, num_chunks, dim=0)\n",
      " |  \n",
      " |  clamp(self, min=None, max=None)\n",
      " |  \n",
      " |  clone(self)\n",
      " |  \n",
      " |  contiguous(self)\n",
      " |  \n",
      " |  cos(self)\n",
      " |  \n",
      " |  cosh(self)\n",
      " |  \n",
      " |  cpu(self)\n",
      " |  \n",
      " |  cross(self, other, dim=-1)\n",
      " |  \n",
      " |  cuda(self, device_id=None, async=False)\n",
      " |  \n",
      " |  cumsum(self, dim)\n",
      " |  \n",
      " |  detach(self)\n",
      " |      Returns a new Variable, detached from the current graph.\n",
      " |      \n",
      " |      Result will never require gradient. If the input is volatile, the output\n",
      " |      will be volatile too.\n",
      " |      \n",
      " |      .. note::\n",
      " |      \n",
      " |        Returned Variable uses the same data tensor, as the original one, and\n",
      " |        in-place modifications on either of them will be seen, and may trigger\n",
      " |        errors in correctness checks.\n",
      " |  \n",
      " |  detach_(self)\n",
      " |      Detaches the Variable from the graph that created it, making it a leaf.\n",
      " |  \n",
      " |  diag(self, diagonal_idx=0)\n",
      " |  \n",
      " |  dist(self, tensor, p=2)\n",
      " |  \n",
      " |  div(self, other)\n",
      " |  \n",
      " |  div_(self, other)\n",
      " |  \n",
      " |  dot(self, other)\n",
      " |  \n",
      " |  double(self)\n",
      " |  \n",
      " |  eq(self, other)\n",
      " |  \n",
      " |  exp(self)\n",
      " |  \n",
      " |  exp_(self)\n",
      " |  \n",
      " |  expand(self, *sizes)\n",
      " |  \n",
      " |  expand_as(self, tensor)\n",
      " |  \n",
      " |  float(self)\n",
      " |  \n",
      " |  floor(self)\n",
      " |  \n",
      " |  fmod(self, value)\n",
      " |  \n",
      " |  frac(self)\n",
      " |  \n",
      " |  gather(self, dim, index)\n",
      " |  \n",
      " |  ge(self, other)\n",
      " |  \n",
      " |  ger(self, vector)\n",
      " |  \n",
      " |  gt(self, other)\n",
      " |  \n",
      " |  half(self)\n",
      " |  \n",
      " |  index_add(self, dim, index, tensor)\n",
      " |  \n",
      " |  index_add_(self, dim, index, tensor)\n",
      " |  \n",
      " |  index_copy(self, dim, index, tensor)\n",
      " |  \n",
      " |  index_copy_(self, dim, index, tensor)\n",
      " |  \n",
      " |  index_fill(self, dim, index, value)\n",
      " |  \n",
      " |  index_fill_(self, dim, index, value)\n",
      " |  \n",
      " |  index_select(self, dim, index)\n",
      " |  \n",
      " |  int(self)\n",
      " |  \n",
      " |  is_same_size(self, other_var)\n",
      " |  \n",
      " |  kthvalue(self, dim)\n",
      " |  \n",
      " |  le(self, other)\n",
      " |  \n",
      " |  lerp(self, tensor, weight)\n",
      " |  \n",
      " |  log(self)\n",
      " |  \n",
      " |  log1p(self)\n",
      " |  \n",
      " |  long(self)\n",
      " |  \n",
      " |  lt(self, other)\n",
      " |  \n",
      " |  masked_copy(self, mask, variable)\n",
      " |  \n",
      " |  masked_copy_(self, mask, variable)\n",
      " |  \n",
      " |  masked_fill(self, mask, value)\n",
      " |  \n",
      " |  masked_fill_(self, mask, value)\n",
      " |  \n",
      " |  masked_select(self, mask)\n",
      " |  \n",
      " |  max(self, dim=None)\n",
      " |  \n",
      " |  mean(self, dim=None)\n",
      " |  \n",
      " |  median(self, dim)\n",
      " |  \n",
      " |  min(self, dim=None)\n",
      " |  \n",
      " |  mm(self, matrix)\n",
      " |  \n",
      " |  mode(self, dim)\n",
      " |  \n",
      " |  mul(self, other)\n",
      " |  \n",
      " |  mul_(self, other)\n",
      " |  \n",
      " |  multinomial(self, num_samples=1, with_replacement=False)\n",
      " |  \n",
      " |  mv(self, vector)\n",
      " |  \n",
      " |  narrow(self, dim, start_index, length)\n",
      " |  \n",
      " |  ne(self, other)\n",
      " |  \n",
      " |  neg(self)\n",
      " |  \n",
      " |  neg_(self)\n",
      " |  \n",
      " |  norm(self, p=2, dim=None)\n",
      " |  \n",
      " |  permute(self, *permutation)\n",
      " |  \n",
      " |  pow(self, other)\n",
      " |  \n",
      " |  prod(self, dim=None)\n",
      " |  \n",
      " |  reciprocal(self)\n",
      " |  \n",
      " |  register_hook(self, hook)\n",
      " |      Registers a backward hook.\n",
      " |      \n",
      " |      The hook will be called every time a gradient with respect to the\n",
      " |      variable is computed. The hook should have the following signature::\n",
      " |      \n",
      " |          hook(grad) -> Variable or None\n",
      " |      \n",
      " |      The hook should not modify its argument, but it can optionally return\n",
      " |      a new gradient which will be used in place of :attr:`grad`.\n",
      " |      \n",
      " |      This function returns a handle with a method ``handle.remove()``\n",
      " |      that removes the hook from the module.\n",
      " |      \n",
      " |      Example:\n",
      " |          >>> v = Variable(torch.Tensor([0, 0, 0]), requires_grad=True)\n",
      " |          >>> h = v.register_hook(lambda grad: grad * 2)  # double the gradient\n",
      " |          >>> v.backward(torch.Tensor([1, 1, 1]))\n",
      " |          >>> v.grad.data\n",
      " |           2\n",
      " |           2\n",
      " |           2\n",
      " |          [torch.FloatTensor of size 3]\n",
      " |          >>> h.remove()  # removes the hook\n",
      " |  \n",
      " |  reinforce(self, reward)\n",
      " |      Registers a reward obtained as a result of a stochastic process.\n",
      " |      \n",
      " |      Differentiating stochastic nodes requires providing them with reward\n",
      " |      value. If your graph contains any stochastic operations, you should\n",
      " |      call this function on their outputs. Otherwise an error will be raised.\n",
      " |      \n",
      " |      Parameters:\n",
      " |          reward(Tensor): Tensor with per-element rewards. It has to match\n",
      " |              the device location and shape of Variable's data.\n",
      " |  \n",
      " |  remainder(self, value)\n",
      " |  \n",
      " |  renorm(self, p, dim, maxnorm)\n",
      " |  \n",
      " |  repeat(self, *repeats)\n",
      " |  \n",
      " |  resize(self, *sizes)\n",
      " |  \n",
      " |  resize_as(self, variable)\n",
      " |  \n",
      " |  round(self)\n",
      " |  \n",
      " |  rsqrt(self)\n",
      " |  \n",
      " |  scatter(self, dim, index, source)\n",
      " |  \n",
      " |  scatter_(self, dim, index, source)\n",
      " |  \n",
      " |  select(self, dim, _index)\n",
      " |  \n",
      " |  short(self)\n",
      " |  \n",
      " |  sigmoid(self)\n",
      " |  \n",
      " |  sigmoid_(self)\n",
      " |  \n",
      " |  sign(self)\n",
      " |  \n",
      " |  sin(self)\n",
      " |  \n",
      " |  sinh(self)\n",
      " |  \n",
      " |  sort(self, dim=None, descending=False)\n",
      " |  \n",
      " |  split(self, split_size, dim=0)\n",
      " |  \n",
      " |  sqrt(self)\n",
      " |  \n",
      " |  squeeze(self, dim=None)\n",
      " |  \n",
      " |  std(self, dim=None, unbiased=True)\n",
      " |  \n",
      " |  sub(self, other)\n",
      " |  \n",
      " |  sub_(self, other)\n",
      " |  \n",
      " |  sum(self, dim=None)\n",
      " |  \n",
      " |  t(self)\n",
      " |  \n",
      " |  tan(self)\n",
      " |  \n",
      " |  tanh(self)\n",
      " |  \n",
      " |  tanh_(self)\n",
      " |  \n",
      " |  topk(self, k, dim=None, largest=True, sorted=True)\n",
      " |  \n",
      " |  trace(self)\n",
      " |  \n",
      " |  transpose(self, dim1, dim2)\n",
      " |  \n",
      " |  tril(self, diagonal_idx=0)\n",
      " |  \n",
      " |  triu(self, diagonal_idx=0)\n",
      " |  \n",
      " |  trunc(self)\n",
      " |  \n",
      " |  type(self, t)\n",
      " |  \n",
      " |  type_as(self, t)\n",
      " |  \n",
      " |  unsqueeze(self, dim)\n",
      " |  \n",
      " |  var(self, dim=None, unbiased=True)\n",
      " |  \n",
      " |  view(self, *sizes)\n",
      " |  \n",
      " |  view_as(self, tensor)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from torch._C._VariableBase:\n",
      " |  \n",
      " |  __init__(self, /, *args, **kwargs)\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  __new__(*args, **kwargs) from builtins.type\n",
      " |      Create and return a new object.  See help(type) for accurate signature.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from torch._C._VariableBase:\n",
      " |  \n",
      " |  creator\n",
      " |  \n",
      " |  data\n",
      " |  \n",
      " |  grad\n",
      " |  \n",
      " |  output_nr\n",
      " |  \n",
      " |  requires_grad\n",
      " |  \n",
      " |  volatile\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(Variable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A bit on NN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MyNN (\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): Linear (400 -> 120)\n",
      "  (fc2): Linear (120 -> 84)\n",
      "  (fc3): Linear (84 -> 10)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as fxnl\n",
    "\n",
    "class MyNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MyNN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1,6,5)\n",
    "        self.conv2 = nn.Conv2d(6,16,5)\n",
    "        # An affine operation y = Wx+b\n",
    "        self.fc1 = nn.Linear(16*5*5, 120)\n",
    "        self.fc2 = nn.Linear(120,84)\n",
    "        self.fc3 = nn.Linear(84,10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # max pooling over a 2x2 window\n",
    "        x = fxnl.max_pool2d(fxnl.relu(self.conv1(x)), (2,2))\n",
    "        x= fxnl.max_pool2d(fxnl.relu(self.conv2(x)), 2)\n",
    "        x= x.view(-1, self.num_flat_features(x))\n",
    "        x = fxnl.relu(self.fc1(x))\n",
    "        x = fxnl.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "    def num_flat_features(self, x):\n",
    "        # all dimensions except the batch dimension\n",
    "        size = x.size()[1:]\n",
    "        num_features = 1\n",
    "        for s in size:\n",
    "            num_features *= s\n",
    "        return num_features\n",
    "\n",
    "\n",
    "net = MyNN()\n",
    "print(net)          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Notes\n",
    "All you need to do is define the forward function and backward function (this is where the gradients - you know the curvatures , or the steepest path to local minima are computed) is automatically defined for us by using autograd.\n",
    "We can use any of the Tensor ops in the forward function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "torch.Size([6, 1, 5, 5])\n"
     ]
    }
   ],
   "source": [
    "params = list(net.parameters())\n",
    "print(len(params))\n",
    "print(params[0].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "-0.0525  0.1221 -0.0390  0.0300  0.0151  0.0359 -0.0839 -0.1024 -0.1495 -0.0095\n",
      "[torch.FloatTensor of size 1x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Input to the forward is autograd.Variable as is the output\n",
    "\n",
    "input = Variable(torch.randn(1, 1, 32, 32))\n",
    "output = net(input)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First backward of  Variable containing:\n",
      "-0.0525  0.1221 -0.0390  0.0300  0.0151  0.0359 -0.0839 -0.1024 -0.1495 -0.0095\n",
      "[torch.FloatTensor of size 1x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "net.zero_grad()\n",
    "output.backward(torch.randn(1, 10), retain_variables=True)\n",
    "print(\"First backward of \", output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarizing\n",
    "\n",
    "1. torch.Tensor: We create a multi-dimensional array with torch.Tensor\n",
    "2. autograd.Variable: W2 wrapped a Tensor and recorded the history of ops applied to it.\n",
    "3. nn.Module - We played with the Neural Network Module\n",
    "4. nn.Parameter - This parameter is automatically registered as a parameter with helpers for moving then to gpu, loading etc\n",
    "5. autograd.Function = This implemented forward and backward definitions of an autograd op.\n",
    "\n",
    "So, all in all we defined a neural network AND we processed inputs and called it backward.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}