pytorch-autograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51e31af9",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "intro to pytorch's `torch.autograd`\n",
    "an automatic differentiation engine that powers nn training\n",
    "\"\"\"\n",
    "\n",
    "import torch, torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0aa86686",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /home/shubham/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
      "70.8%IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100.0%\n"
     ]
    }
   ],
   "source": [
    "model = torchvision.models.resnet18(pretrained=True)\n",
    "data = torch.rand(1, 3, 64, 64)\n",
    "labels = torch.rand(1, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "08901bfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = model(data) # forward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "38ae9fb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = (prediction - labels).sum()\n",
    "loss.backward() # backward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c1af61a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "optim = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8d82a80d",
   "metadata": {},
   "outputs": [],
   "source": [
    "optim.step() # gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e9f6c17b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Differentiation in Autograd\n",
    "\n",
    "a = torch.tensor([2., 3.], requires_grad=True)\n",
    "b = torch.tensor([6., 4.], requires_grad=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "689e044f",
   "metadata": {},
   "source": [
    "We create another tensor ``Q`` from ``a`` and ``b``.\n",
    "\n",
    "\\begin{align}Q = 3a^3 - b^2\\end{align}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d79a5189",
   "metadata": {},
   "outputs": [],
   "source": [
    "Q = 3*a**3 - b**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bf1af1ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "external_grad = torch.tensor([1., 1.])\n",
    "Q.backward(gradient=external_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c5e4660b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([True, True])\n",
      "tensor([True, True])\n"
     ]
    }
   ],
   "source": [
    "# check if collected gradients are correct\n",
    "print(9*a**2 == a.grad)\n",
    "print(-2*b == b.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9209bda",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}