mnist_neural_net.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: numpy in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (1.26.4)\n",
      "Requirement already satisfied: pandas in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (2.2.2)\n",
      "Requirement already satisfied: matplotlib in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (3.9.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (1.3.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (4.54.1)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (1.4.7)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (24.0)\n",
      "Requirement already satisfied: pillow>=8 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (11.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: six>=1.5 in /Users/figredos/Documents/unb/.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# Installing requirements\n",
    "%pip install numpy pandas matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing data\n",
    "mnist_data = np.array(pd.read_csv('dataset/handwritten_numbers.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting data's shape\n",
    "m, n = mnist_data.shape\n",
    "\n",
    "# Shuffling data\n",
    "np.random.shuffle(mnist_data)\n",
    "\n",
    "# Separating test data (10% of total)\n",
    "test_data = mnist_data[:4200].T\n",
    "Y_test = test_data[0]\n",
    "X_test = test_data[1:n]\n",
    "\n",
    "# Separating training data\n",
    "train_data = mnist_data[4200:m].T\n",
    "Y_train = train_data[0]\n",
    "X_train = train_data[1:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalizing data\n",
    "X_test = X_test / 255\n",
    "X_train = X_train / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining activation functions\n",
    "\n",
    "\n",
    "# ReLU activation function\n",
    "def relu(Z):\n",
    "    return np.maximum(0, Z)\n",
    "\n",
    "\n",
    "# Derivative of ReLU\n",
    "def d_relu_dx(Z):\n",
    "    return Z > 0\n",
    "\n",
    "\n",
    "# Softmax\n",
    "def softmax(Z):\n",
    "    return np.exp(Z) / np.sum(np.exp(Z), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Neural net class\n",
    "class MnistNeuralNet:\n",
    "    def __init__(self, X: np.ndarray, Y: np.ndarray, alpha: float) -> None:\n",
    "        # Data for neural net\n",
    "        self.X = X\n",
    "        self.Y = Y\n",
    "\n",
    "        # Learning rate\n",
    "        self.alpha = alpha\n",
    "\n",
    "    def init_params(self) -> None:\n",
    "        # First layers weights and biases\n",
    "        self.W1 = np.random.rand(10, 784) - 0.5\n",
    "        self.b1 = np.random.rand(10, 1) - 0.5\n",
    "\n",
    "        # Second layers weights and biases\n",
    "        self.W2 = np.random.rand(10, 10) - 0.5\n",
    "        self.b2 = np.random.rand(10, 1) - 0.5\n",
    "\n",
    "    def forward_propagation(self, X: np.ndarray = None) -> None:\n",
    "        # Setting what data to predict on (for training / predicting)\n",
    "        X_data: np.ndarray = self.X if X is None else X\n",
    "\n",
    "        # First layer nerons output and activation\n",
    "        self.Z1 = self.W1.dot(X_data) + self.b1\n",
    "        self.A1 = relu(self.Z1)\n",
    "\n",
    "        # Second layer nerons output and activation\n",
    "        self.Z2 = self.W2.dot(self.A1) + self.b2\n",
    "        self.A2 = softmax(self.Z2)\n",
    "\n",
    "    def back_propagation(self) -> list[np.ndarray]:\n",
    "        # Creating one hot encoded array\n",
    "        one_hot_Y = self.one_hot()\n",
    "\n",
    "        # Derivative of last layer neurons, weights and biases\n",
    "        dZ2 = (1 / m) * (self.A2 - one_hot_Y)\n",
    "        dW2 = dZ2.dot(self.A1.T)\n",
    "        db2 = np.sum(dZ2, axis=1).reshape(-1, 1)\n",
    "\n",
    "        # Derivative of first layer neurons, weights and biases\n",
    "        dZ1 = self.W2.T.dot(dZ2) * d_relu_dx(self.Z1)\n",
    "        dW1 = dZ1.dot(self.X.T)\n",
    "        db1 = np.sum(dZ1, axis=1).reshape(-1, 1)\n",
    "\n",
    "        # Returning derivative values\n",
    "        return dW1, db1, dW2, db2\n",
    "\n",
    "    def update_params(\n",
    "        self, dW1: np.ndarray, db1: np.ndarray, dW2: np.ndarray, db2: np.ndarray\n",
    "    ) -> None:\n",
    "        # Updating all weights and biases\n",
    "        self.W1 -= self.alpha * dW1\n",
    "        self.b1 -= self.alpha * db1\n",
    "        self.W2 -= self.alpha * dW2\n",
    "        self.b2 -= self.alpha * db2\n",
    "\n",
    "    def gradient_descent(self, epochs: int) -> None:\n",
    "        # Repeating process over a number of epochs\n",
    "        for i in range(epochs):\n",
    "            # Doing forward propagation\n",
    "            self.forward_propagation()\n",
    "\n",
    "            # Obtaining derivatives\n",
    "            dW1, db1, dW2, db2 = self.back_propagation()\n",
    "\n",
    "            self.update_params(dW1, db1, dW2, db2)\n",
    "\n",
    "            # Showing progress and accuracy every 50 iterations\n",
    "            if i % 50 == 0:\n",
    "                print('Iteration: ', i)\n",
    "                predictions = self.get_predictions()\n",
    "                print(self.get_accuracy(predictions))\n",
    "\n",
    "    def one_hot(self) -> np.ndarray:\n",
    "        # Turning Y values into int\n",
    "        temp_Y = self.Y.astype(int)\n",
    "\n",
    "        # Creating empty array\n",
    "        one_hot_Y = np.zeros((temp_Y.size, temp_Y.max() + 1))\n",
    "\n",
    "        # Turning value at each max to 1\n",
    "        one_hot_Y[np.arange(temp_Y.size), temp_Y] = 1\n",
    "\n",
    "        return one_hot_Y.T\n",
    "\n",
    "    def get_predictions(self) -> np.ndarray:\n",
    "        # Returns the maximum value in output array or 0\n",
    "        return np.argmax(self.A2, 0)\n",
    "\n",
    "    def get_accuracy(self, predictions) -> np.ndarray:\n",
    "        # Returns number of correct predictions over array's size\n",
    "        return np.sum(predictions == self.Y) / self.Y.size\n",
    "\n",
    "    def make_predictions(self, X) -> np.ndarray:\n",
    "        # Doing forward propagation for given data\n",
    "        self.forward_propagation(X)\n",
    "\n",
    "        # Returning predictions\n",
    "        predictions = self.get_predictions()\n",
    "        return predictions\n",
    "\n",
    "    def test_prediction(self, index: int) -> None:\n",
    "        # Getting image that is being predicted on\n",
    "        current_image = X_test[:, index, None]\n",
    "\n",
    "        # Getting prediction\n",
    "        prediction = self.make_predictions(X_test[:, index, None])\n",
    "\n",
    "        # Getting correct label\n",
    "        label = Y_test[index]\n",
    "\n",
    "        # Printing and showing image\n",
    "        print('Prediction: ', prediction)\n",
    "        print('Label: ', label)\n",
    "\n",
    "        current_image = current_image.reshape(28, 28)\n",
    "        plt.imshow(current_image, cmap='gray')\n",
    "        plt.axis('off')\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:  0\n",
      "0.1041005291005291\n",
      "Iteration:  50\n",
      "0.36873015873015874\n",
      "Iteration:  100\n",
      "0.5966137566137566\n",
      "Iteration:  150\n",
      "0.686005291005291\n",
      "Iteration:  200\n",
      "0.7369576719576719\n",
      "Iteration:  250\n",
      "0.7705291005291005\n",
      "Iteration:  300\n",
      "0.7935449735449736\n",
      "Iteration:  350\n",
      "0.8103968253968254\n",
      "Iteration:  400\n",
      "0.8230687830687831\n",
      "Iteration:  450\n",
      "0.8312962962962963\n"
     ]
    }
   ],
   "source": [
    "# Instantiating class\n",
    "neural_net = MnistNeuralNet(X=X_train, Y=Y_train, alpha=0.1)\n",
    "\n",
    "# Initializing parameters\n",
    "neural_net.init_params()\n",
    "\n",
    "# Training model\n",
    "neural_net.gradient_descent(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction:  [9]\n",
      "Label:  9\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAI5klEQVR4nO3cz4vNfR/H8e/3bhJTNkrsdNmSFFIWNiIbkxpSpPwoS2WhiD9ASkpRKM1YWFiwkI3YGOIPsJKkWUlThCTR997cvbqv7qvu8/5ejjnXeDzW59X5pOk8+yx82q7rugYAmqb513wfAIDRIQoAhCgAEKIAQIgCACEKAIQoABCiAECMDfrBtm2HeQ4AhmyQ/6vspgBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCADE23weAYdi8eXN5s3v37vJmzZo15c2uXbvKm6Zpmq7rypsbN26UNydPnixv3r9/X94wmtwUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAKLtBnxlq23bYZ+FBe7UqVO9dvv37y9v+jxU18e3b9/Km+/fvw/hJH9tyZIl5c3MzEx5MzExUd58+vSpvOHvGeTn3k0BgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgPBKKs3ixYvLmwMHDpQ3V65cKW+apmnGxsbKm5cvX5Y3Dx8+LG8uXLhQ3rx586a86Wtqaqq8OXjwYHlz5syZ8ubcuXPlDX+PV1IBKBEFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIOovjbHgTExMlDfXr18vb75//17eNE2/x9b6PL738ePH8mbUXbx4sbzp8yAeC4ebAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAEB4EG+B2bhxY3kzNTVV3nRdV95MTk6WN03TNPfu3eu1o2kuXbo030fgH8ZNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACA8iDeiVqxY0Wt39erV8mZsrP5ncPr06fLm/v375c1CtHjx4l67y5cvlzdbtmzp9V1Vnz9//iXfw/C5KQAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQXkkdUUeOHOm1W79+fXkzOztb3pw/f768WYiWL19e3pw4caLXdx06dKi8adu2vJmZmSlvpqenyxtGk5sCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQHgQb0R1XTffRxgZq1atKm+WLFlS3kxOTpY3p06dKm/Gx8fLm776PHZ4+PDh8ubTp0/lDaPJTQGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgPIg3otq2/WW7lStXljePHz8ub/pat25debN06dIhnOR/9fn3/pWPHZ49e7a8ef369RBOwj+FmwIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAtN2Ar3P1faCNfsbHx3vtbt68Wd7s2rWrvHnz5k158/Hjx/Kmafr97T158qS8uXv3bnnT52yPHj0qb/r6448/ypvZ2dkhnIRRMMjPvZsCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIzN9wH4a1++fOm127NnT3mzdu3a8qbPo2l9H8QbZceOHftl3/X8+fPy5u3bt0M4CQuZmwIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIA4ZVUmhcvXsz3EUbCtm3bypuTJ0+WNz9+/ChvmqZppqeny5tv3771+i5+X24KAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCAOFBPPiPycnJ8mb16tXlzdzcXHnTNE1z7dq1XjuocFMAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACA/isSBt3bq1vNm3b1958+HDh/Jm79695Q38Km4KAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCANF2XdcN9MG2HfZZ4Kd59uxZebN58+by5sGDB+XNzp07yxv4GQb5uXdTACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAIix+T4A/D9Hjx4tbzZt2lTeDPg25J/cunWrvIFR5qYAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQLTdgE9Dtm077LOwwC1atKjX7unTp+XNhg0bfsn3bN++vbz5+vVreQM/wyA/924KAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCADE23wfg97Fjx45euz6P2w34zuOf3Llzp7zxuB0LjZsCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQLTdgC+HtW077LOwwM3NzfXaLVu2rLy5fft2eXP8+PHy5t27d+UNzJdBfu7dFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBibL4PwO/j1atXvXabNm0qb65evVreeNwO3BQA+C+iAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABBt13XdQB9s22GfBYAhGuTn3k0BgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBgb9INd1w3zHACMADcFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUA4t/8PxxvUZH+ggAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Getting data's shape\n",
    "a, b = X_test.shape\n",
    "\n",
    "# Making predictions\n",
    "neural_net.test_prediction(np.random.randint(1, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}