notebooks/lin_reg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LinReg with PyTorch, Gradient Descent, and GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init, helpers, utils ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "\n",
    "DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_regression\n",
    "\n",
    "\n",
    "n_features = 1\n",
    "n_samples = 100\n",
    "\n",
    "X, y = make_regression(\n",
    "    n_samples=n_samples,\n",
    "    n_features=n_features,\n",
    "    noise=20,\n",
    "    random_state=42,\n",
    ")\n",
    "\n",
    "fix, ax = plt.subplots()\n",
    "ax.plot(X, y, \".\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.from_numpy(X).float()\n",
    "y = torch.from_numpy(y.reshape((n_samples, n_features))).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinReg(nn.Module):\n",
    "    def __init__(self, input_dim):\n",
    "        super().__init__()\n",
    "        self.beta = nn.Linear(input_dim, 1)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return self.beta(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LinReg(n_features).to(DEVICE)  # <-- here\n",
    "loss_fn = nn.MSELoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "\n",
    "\n",
    "X, y = X.to(DEVICE), y.to(DEVICE)  # <-- here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train step\n",
    "model.train()  # <-- here\n",
    "optimizer.zero_grad()\n",
    "\n",
    "y_ = model(X)\n",
    "loss = loss_fn(y_, y)\n",
    "\n",
    "loss.backward()\n",
    "optimizer.step()\n",
    "\n",
    "# Eval\n",
    "model.eval()  # <-- here\n",
    "with torch.no_grad():\n",
    "    y_ = model(X)    \n",
    "\n",
    "# Vis\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(X.cpu().numpy(), y_.cpu().numpy(), \".\", label=\"pred\")\n",
    "ax.plot(X.cpu().numpy(), y.cpu().numpy(), \".\", label=\"data\")\n",
    "ax.set_title(f\"MSE: {loss.item():0.1f}\")\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: I did gradient descent with all the data. I did not split the data into `train` and `valid` which should be done!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(range(100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise:\n",
    "- Write a proper training loop for this linear regression example.\n",
    "- Split data into train and valid.\n",
    "- Use the Dataset and DataLoader abstraction.\n",
    "- Create a logistic regression module.\n",
    "- Create a Multi Layer Perceptron (MLP)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}