02 SGD, BGD, mini-BGD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stochastic, mini-Batch, Batch Gradient Descent and Dataloader\n",
    "\n",
    "In this notebook, a generic dataset loader that enables mini-Batch and Stochastic Gradient Descent is implemented. The keypoints of this experiment are the following:\n",
    "\n",
    "* Assuming a __non-parallel implementation__, __Batch Gradient Descent (BGD) calculates the fastest__ among the three.\n",
    "* Batch Gradient Descent's calculating the fastest among the three does not necessarily mean that it also converges the fastest to the minimum. \n",
    "* Smaller learning rate values is needed for smaller batch sizes\n",
    "* __Stochastic Gradient Descent (SGD)__ is a magnitude slower than the two, and therefore, it is omitted in this experiment. \n",
    "* __Mini-batch Gradient Descent__ is the better compromise between the three. A __good choice of batch size and learning rate__ may produce good performance, while also minimizing the time and computational cost of training a model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import mnist\n",
    "from activation import relu, sigmoid, sigmoid_prime, softmax\n",
    "from helper import one_hot_encoder\n",
    "from initializer import initialize_weight\n",
    "%matplotlib inline\n",
    "\n",
    "from utils import dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_x, train_y, learning_rate=0.1, num_epochs=50, batch_size=1):\n",
    "    # Flatten input (num_samples, 28, 28) -> (num_samples, 784) \n",
    "    x = train_x.reshape(train_x.shape[0], -1)\n",
    "    num_samples = x.shape[0]\n",
    "    \n",
    "    # Turn labels into their one-hot representations\n",
    "    y = one_hot_encoder(train_y)\n",
    "\n",
    "    # Make a data loader\n",
    "    trainloader = dataloader(x, y, batch_size=batch_size, shuffle=True)\n",
    "    \n",
    "    # Initialize weights\n",
    "    w1, b1 = initialize_weight((784, 256), bias=True)\n",
    "    w2, b2 = initialize_weight((256, 10), bias=True)\n",
    "\n",
    "    loss_history = []\n",
    "    for epoch in range(1, num_epochs+1):\n",
    "        print(\"Epoch {}/{}\\n===============\".format(epoch, num_epochs))\n",
    "\n",
    "        batch_loss = 0\n",
    "        acc = 0\n",
    "        for inputs, labels in trainloader:\n",
    "            # Number of samples per batch\n",
    "            m = inputs.shape[0]\n",
    "            \n",
    "            # Forward Prop\n",
    "            h1 = np.dot(inputs, w1) + b1\n",
    "            a1 = sigmoid(h1)\n",
    "            h2 = np.dot(a1, w2) + b2\n",
    "            a2 = softmax(h2)\n",
    "            out = a2\n",
    "\n",
    "            # Cross Entropy Loss\n",
    "            batch_loss += cross_entropy_loss(out, labels.argmax(axis=1).reshape(m,1))\n",
    "\n",
    "            # Compute Accuracy\n",
    "            pred = np.argmax(out, axis=1)\n",
    "            pred = pred.reshape(pred.shape[0], 1)\n",
    "            acc += np.sum(pred == labels.argmax(axis=1).reshape(m,1))\n",
    "\n",
    "            # Backward Prop\n",
    "            dh2 = a2 - labels \n",
    "            dw2 = (1/m) * np.dot(a1.T, dh2)\n",
    "            db2 = (1/m) * np.sum(dh2, axis=0, keepdims=True)\n",
    "\n",
    "            dh1 = np.dot(dh2, w2.T) * sigmoid_prime(a1)\n",
    "            dw1 = (1/m) * np.dot(inputs.T, dh1)\n",
    "            db1 = (1/m) * np.sum(dh1, axis=0, keepdims=True)\n",
    "\n",
    "            # Weight (and bias) update\n",
    "            w1 -= learning_rate * dw1\n",
    "            b1 -= learning_rate * db1\n",
    "            w2 -= learning_rate * dw2\n",
    "            b2 -= learning_rate * db2\n",
    "            \n",
    "        loss_history.append(batch_loss/num_samples)\n",
    "        print(\"Loss: {:.6f}\".format(batch_loss/num_samples))\n",
    "        print(\"Accuracy: {:.2f}%\\n\".format(acc/num_samples*100))\n",
    "\n",
    "    return w1, b1, w2, b2, loss_history\n",
    "\n",
    "def cross_entropy_loss(out, y):\n",
    "    batch_size = y.shape[0]\n",
    "    y = y.reshape(batch_size)\n",
    "    log_likelihood = -np.log(out[np.arange(batch_size), y])\n",
    "    return np.sum(log_likelihood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified data/t10k-images-idx3-ubyte.gz\n",
      "Found and verified data/t10k-labels-idx1-ubyte.gz\n",
      "Found and verified data/train-images-idx3-ubyte.gz\n",
      "Found and verified data/train-labels-idx1-ubyte.gz\n",
      "Found and verified data/t10k-images-idx3-ubyte.gz\n",
      "Found and verified data/t10k-labels-idx1-ubyte.gz\n",
      "Found and verified data/train-images-idx3-ubyte.gz\n",
      "Found and verified data/train-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Load dataset\n",
    "train_x, train_y = mnist.load_dataset(download=True, train=True)\n",
    "test_x, test_y = mnist.load_dataset(download=True, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "===============\n",
      "Loss: 2.507978\n",
      "Accuracy: 9.36%\n",
      "\n",
      "Epoch 2/20\n",
      "===============\n",
      "Loss: 2.119753\n",
      "Accuracy: 33.29%\n",
      "\n",
      "Epoch 3/20\n",
      "===============\n",
      "Loss: 1.771465\n",
      "Accuracy: 47.28%\n",
      "\n",
      "Epoch 4/20\n",
      "===============\n",
      "Loss: 1.609259\n",
      "Accuracy: 50.66%\n",
      "\n",
      "Epoch 5/20\n",
      "===============\n",
      "Loss: 1.409475\n",
      "Accuracy: 57.37%\n",
      "\n",
      "Epoch 6/20\n",
      "===============\n",
      "Loss: 1.212161\n",
      "Accuracy: 63.29%\n",
      "\n",
      "Epoch 7/20\n",
      "===============\n",
      "Loss: 1.110838\n",
      "Accuracy: 68.16%\n",
      "\n",
      "Epoch 8/20\n",
      "===============\n",
      "Loss: 1.006603\n",
      "Accuracy: 70.79%\n",
      "\n",
      "Epoch 9/20\n",
      "===============\n",
      "Loss: 0.924600\n",
      "Accuracy: 76.35%\n",
      "\n",
      "Epoch 10/20\n",
      "===============\n",
      "Loss: 0.856999\n",
      "Accuracy: 76.07%\n",
      "\n",
      "Epoch 11/20\n",
      "===============\n",
      "Loss: 0.806822\n",
      "Accuracy: 80.19%\n",
      "\n",
      "Epoch 12/20\n",
      "===============\n",
      "Loss: 0.788305\n",
      "Accuracy: 76.78%\n",
      "\n",
      "Epoch 13/20\n",
      "===============\n",
      "Loss: 0.767588\n",
      "Accuracy: 79.50%\n",
      "\n",
      "Epoch 14/20\n",
      "===============\n",
      "Loss: 0.773671\n",
      "Accuracy: 75.31%\n",
      "\n",
      "Epoch 15/20\n",
      "===============\n",
      "Loss: 0.753177\n",
      "Accuracy: 78.37%\n",
      "\n",
      "Epoch 16/20\n",
      "===============\n",
      "Loss: 0.716950\n",
      "Accuracy: 77.40%\n",
      "\n",
      "Epoch 17/20\n",
      "===============\n",
      "Loss: 0.679173\n",
      "Accuracy: 81.19%\n",
      "\n",
      "Epoch 18/20\n",
      "===============\n",
      "Loss: 0.632208\n",
      "Accuracy: 81.63%\n",
      "\n",
      "Epoch 19/20\n",
      "===============\n",
      "Loss: 0.601936\n",
      "Accuracy: 84.41%\n",
      "\n",
      "Epoch 20/20\n",
      "===============\n",
      "Loss: 0.573346\n",
      "Accuracy: 84.83%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Batch Gradient Descent\n",
    "w1_bgd, b1_bgd, w2_bgd, b2_bgd, loss_history_bgd = train(train_x, train_y, learning_rate=0.2, num_epochs=20, batch_size=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "===============\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rusty\\Desktop\\mnist_experiments\\activation.py:9: RuntimeWarning: overflow encountered in exp\n",
      "  return 1/(1+np.exp(-x))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.560291\n",
      "Accuracy: 84.58%\n",
      "\n",
      "Epoch 2/20\n",
      "===============\n",
      "Loss: 0.407835\n",
      "Accuracy: 88.56%\n",
      "\n",
      "Epoch 3/20\n",
      "===============\n",
      "Loss: 0.392708\n",
      "Accuracy: 88.81%\n",
      "\n",
      "Epoch 4/20\n",
      "===============\n",
      "Loss: 0.384136\n",
      "Accuracy: 88.86%\n",
      "\n",
      "Epoch 5/20\n",
      "===============\n",
      "Loss: 0.388819\n",
      "Accuracy: 88.49%\n",
      "\n",
      "Epoch 6/20\n",
      "===============\n",
      "Loss: 0.375824\n",
      "Accuracy: 88.98%\n",
      "\n",
      "Epoch 7/20\n",
      "===============\n",
      "Loss: 0.379130\n",
      "Accuracy: 88.77%\n",
      "\n",
      "Epoch 8/20\n",
      "===============\n",
      "Loss: 0.369271\n",
      "Accuracy: 88.95%\n",
      "\n",
      "Epoch 9/20\n",
      "===============\n",
      "Loss: 0.361980\n",
      "Accuracy: 89.11%\n",
      "\n",
      "Epoch 10/20\n",
      "===============\n",
      "Loss: 0.349849\n",
      "Accuracy: 89.60%\n",
      "\n",
      "Epoch 11/20\n",
      "===============\n",
      "Loss: 0.338091\n",
      "Accuracy: 89.63%\n",
      "\n",
      "Epoch 12/20\n",
      "===============\n",
      "Loss: 0.345172\n",
      "Accuracy: 89.54%\n",
      "\n",
      "Epoch 13/20\n",
      "===============\n",
      "Loss: 0.341380\n",
      "Accuracy: 89.72%\n",
      "\n",
      "Epoch 14/20\n",
      "===============\n",
      "Loss: 0.334485\n",
      "Accuracy: 89.84%\n",
      "\n",
      "Epoch 15/20\n",
      "===============\n",
      "Loss: 0.323573\n",
      "Accuracy: 90.17%\n",
      "\n",
      "Epoch 16/20\n",
      "===============\n",
      "Loss: 0.330119\n",
      "Accuracy: 89.87%\n",
      "\n",
      "Epoch 17/20\n",
      "===============\n",
      "Loss: 0.337218\n",
      "Accuracy: 89.62%\n",
      "\n",
      "Epoch 18/20\n",
      "===============\n",
      "Loss: 0.310571\n",
      "Accuracy: 90.55%\n",
      "\n",
      "Epoch 19/20\n",
      "===============\n",
      "Loss: 0.309920\n",
      "Accuracy: 90.64%\n",
      "\n",
      "Epoch 20/20\n",
      "===============\n",
      "Loss: 0.296692\n",
      "Accuracy: 91.08%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Minibatch Gradient Descent with batch_size 64\n",
    "w1_mbgd, b1_mbgd, w2_mbgd, b2_mbgd, loss_history_mbgd = train(train_x, train_y, learning_rate=0.1, num_epochs=20, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x14a80597a58>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPW9//HXd7JN9pANAmEJEMAsJISALMoiClgUFfeisoiISmvvr1K1tWq9trcqtbfiVURF0GpFwYVWsUBlkz1A2CEQ1hAIIZB9n3x/f5zJkITJAlkmmXyej8d5zJlzzsz55DC8z/49SmuNEEII52JydAFCCCGanoS7EEI4IQl3IYRwQhLuQgjhhCTchRDCCUm4CyGEE5JwF0IIJyThLoQQTkjCXQghnJCro2YcHByse/To4ajZCyFEm7Rjx44LWuuQ+qZzWLj36NGDpKQkR81eCCHaJKXUyYZMJ4dlhBDCCUm4CyGEE5JwF0IIJ+SwY+5CNLWysjLS0tIoLi52dClCNJrZbCY8PBw3N7dr+ryEu3AaaWlp+Pr60qNHD5RSji5HiGumtSYrK4u0tDQiIiKu6TvqPSyjlOqqlFqjlDqolNqvlHrazjSjlFI5Sqlka/fiNVUjRCMUFxcTFBQkwS7aPKUUQUFBjdoLbciWeznwa631TqWUL7BDKbVKa32gxnQbtNa3XXMlQjQBCXbhLBr7W653y11rfVZrvdPanwccBLo0aq6NcCwzn1f+eYAyS4WjShBCiFbvqq6WUUr1AAYAW+2MHqqU2q2UWqGUim6C2uw6kVXAwo3H+dee9OaahRDXzMXFhfj4eOLi4khISGDTpk11Tp+dnc0777xT7/eOGjWqQTf9HTlyhNtuu41evXoxcOBARo8ezfr16xtcvz1Tp05l6dKlAMyYMYMDB2rutDfM2rVra10eixYtIiQkhAEDBhAZGcm4cePqXXbNLTk5me+//96hNTRGg8NdKeUDLAN+pbXOrTF6J9Bdax0HzAO+qeU7ZiqlkpRSSZmZmddU8Kg+ofTp6MN7644hD/cWrY2npyfJycns3r2b//mf/+H555+vc/qGhntDFBcXM2HCBGbOnElqaio7duxg3rx5HDt27Ippy8vLr2keH3zwAVFRUdf02brCHeD+++9n165dHDlyhOeee45JkyZx8ODBa5pXU2gX4a6UcsMI9k+11l/VHK+1ztVa51v7vwfclFLBdqZboLVO1FonhoTU2zSC/YJNipkjenHoXB5rU65tBSFES8jNzaVDhw4A5OfnM2bMGBISEoiNjeXbb78F4LnnniM1NZX4+HjmzJkDwOuvv05sbCxxcXE899xztu/78ssvGTx4MH369GHDhg1XzO/TTz9l6NChTJw40TYsJiaGqVOnAvDyyy8zc+ZMxo4dyyOPPMKJEye48cYbSUhIqLaXobVm9uzZREVFMWHCBM6fP2/7vqp7ECtXrmTo0KEkJCRw7733kp+fDxhNi7z00ku2v/XQoUOcOHGC+fPn89e//pX4+Hi79Vc1evRoZs6cyYIFCwBITU1l/PjxDBw4kBtvvJFDhw7ZlklMTAxxcXGMGDECAIvFwjPPPENsbCz9+/dn3rx5AOzYsYORI0cycOBAxo0bx9mzZ21/07PPPltt2ZaWlvLiiy+yZMkS4uPjWbJkSd3/2K1QvSdUlXFU/0PgoNb6zVqm6QRkaK21Umowxkojq0krrWJiXGfm/vsw761LZXTf0OaajWjD/vDP/RxIr7mD2ThRnf146fa6jzgWFRURHx9PcXExZ8+e5ccffwSMa5a//vpr/Pz8uHDhAkOGDGHixIn8+c9/Zt++fSQnJwOwYsUKvvnmG7Zu3YqXlxcXL160fXd5eTnbtm3j+++/5w9/+AOrV6+uNu/9+/eTkJBQZ307duzgp59+wtPTk8LCQlatWoXZbObIkSM8+OCDJCUl8fXXX3P48GH27t1LRkYGUVFRTJ8+vdr3XLhwgVdffZXVq1fj7e3Na6+9xptvvsmLLxoXygUHB7Nz507eeecd5s6dywcffMCsWbPw8fHhmWeeadDyTkhI4L333gNg5syZzJ8/n8jISLZu3cqTTz7Jjz/+yCuvvMK///1vunTpQnZ2NgALFizg+PHj7Nq1C1dXVy5evEhZWRm/+MUv+PbbbwkJCWHJkiX87ne/Y+HChbUu21deeYWkpCTefvvtBtXb2jTkapnhwMPAXqVUsnXYb4FuAFrr+cA9wBNKqXKgCHhAN+MxE3dXE4/eEMEfvz9I8uls4rsGNNeshLgqlYdlADZv3swjjzzCvn370Frz29/+lvXr12MymThz5gwZGRlXfH716tVMmzYNLy8vAAIDA23jJk2aBMDAgQM5ceJEvbXcddddHDlyhD59+vDVV8YO98SJE/H09ASMm75mz55NcnIyLi4upKSkALB+/XoefPBBXFxc6Ny5MzfddNMV371lyxYOHDjA8OHDASgtLWXo0KF2a62c99WqjJD8/Hw2bdrEvffeaxtXUlICwPDhw5k6dSr33XefbZ6rV69m1qxZuLoa8RYYGMi+ffvYt28ft9xyC2Bs3YeFhdmttyHLti2oN9y11j8BdV6To7V+G2jR1duD13fjrR+PsGB9Ku9MHtiSsxZtQH1b2C1h6NChXLhwgczMTL7//nsyMzPZsWMHbm5u9OjRw+41zFrrWi+B8/DwAIyTtvaOmUdHR1c7efr111+TlJRUbUvZ29vb1v/Xv/6Vjh07snv3bioqKjCbzbZx9V2Gp7Xmlltu4R//+Mc11doQu3bt4rrrrqOiooKAgADbSrOq+fPns3XrVr777jvi4+NJTk62uwy11kRHR7N58+Zmq7e1abNty/h4uPLwkO6s2HeOExcKHF2OEFc4dOgQFouFoKAgcnJyCA0Nxc3NjTVr1nDypNFqq6+vL3l5ebbPjB07loULF1JYWAhQ7bBMfX7+85+zceNGli9fbhtW+T325OTkEBYWhslk4pNPPsFisQAwYsQIPv/8cywWC2fPnmXNmjVXfHbIkCFs3LiRo0eP2uZTueVfm5p/a13WrVvHggULeOyxx/Dz8yMiIoIvv/wSMIJ69+7dgHEs/vrrr+eVV14hODiY06dPM3bsWObPn28L6YsXL9K3b18yMzNt4V5WVsb+/fubrN7WqM2GO8DU4T1wM5lYsOHKqwGEcITKY+7x8fHcf//9LF68GBcXFyZPnkxSUhKJiYl8+umn9OvXD4CgoCCGDx9OTEwMc+bMYfz48UycOJHExETi4+OZO3dug+ft6enJv/71L+bPn0/Pnj0ZOnQor776Ki+88ILd6Z988kkWL17MkCFDSElJsW3V33XXXURGRhIbG8sTTzzByJEjr/hsSEgIixYt4sEHH6R///4MGTLEdpKzNrfffjtff/11rSdUK09e9unThz/96U8sW7aM6667DjBOFn/44YfExcURHR1tOyE9Z84cYmNjiYmJYcSIEcTFxTFjxgy6detG//79iYuL47PPPsPd3Z2lS5fy7LPPEhcXR3x8fL2XWo4ePZoDBw602ROqylGXEyYmJuqmeFjH81/tYdnOM2x89iZCfD2aoDLRVh08eNAWBkI4A3u/aaXUDq11Yn2fbdNb7gCP3diTMksFizedcHQpQgjRarT5cO8Z4sO4qE58vPkEBSXOcSJECCEaq82HO8DjI3uSW1zO59tPO7oUIYRoFZwi3Ad068DgiEA+3HBMGhQTQgicJNwBZo3sSXpOMf/cLQ2KCSGE04S7NCgmhBCXOU24VzYodjhDGhQTrdvy5cv585//XOc06enp3HPPPXbHTZ06lYiICOLj4+nXrx9/+MMf6p3nokWLSE+ve6920aJFzJ49u97vKi8v57e//S2RkZG2a/r/+Mc/1vu5uqxdu5bbbjOe9dOQ5VOb+lrZrGySOTo6mri4ON58800qKhx7KPdPf/pTs3yv04Q7GA2KhfmbeW9dqqNLEaJWEydOrNbaoz2dO3e2taFuzxtvvEFycjLJycksXryY48eP1/l9DQn3hnrhhRdIT09n7969JCcns2HDBsrKyq6YTmt9TcHZkOVTm/rCvbLtn/3797Nq1SpbQ2GOJOHeAJUNim05dpHk09mOLke0MydOnKBfv37MmDGDmJgYJk+ezOrVqxk+fDiRkZFs27YNqL6FPHXqVH75y18ybNgwevbsaQv0EydOEBMTU+88K9unqby79JVXXmHQoEHExMQwc+ZMtNYsXbqUpKQkJk+eTHx8PEVFRWzfvp1hw4YRFxfH4MGDbbfZp6enM378eCIjI/nNb35zxfwKCwt5//33mTdvnq0tGl9fX15++WVb3ddddx1PPvkkCQkJnD59mieeeILExESio6N56aWXbN/1ww8/0K9fP2644YZqjYtVXT6ZmZncfffdDBo0iEGDBrFx40bAaL54+vTpjBo1ip49e/LWW28B9ptQrk1oaCgLFizg7bffRmuNxWJhzpw5DBo0iP79+9tapDx79iwjRowgPj6emJgY2921P/zwAwkJCcTFxTFmzBgACgoKmD59OoMGDWLAgAG2O2kXLVrEpEmTrli2zz33nO2u5smTJ9f7731VtNYO6QYOHKibQ15xmY556Qf9xN+TmuX7Ret14MCBy2++f1brhT9r2u77Z+uc//Hjx7WLi4ves2ePtlgsOiEhQU+bNk1XVFTob775Rt9xxx1aa60/+ugj/dRTT2mttZ4yZYq+5557tMVi0fv379e9evWyfVd0dLTd+UyZMkX36NFDx8XFaW9vb/3888/bxmVlZdn6H3roIb18+XKttdYjR47U27dv11prXVJSoiMiIvS2bdu01lrn5OTosrIy/dFHH+mIiAidnZ2ti4qKdLdu3fSpU6eqzXv37t06Pj6+zmWglNKbN2++oqby8nI9cuRIvXv3bl1UVKTDw8N1SkqKrqio0Pfee6+eMGHCFcvnwQcf1Bs2bNBaa33y5Endr18/rbXWL730kh46dKguLi7WmZmZOjAwUJeWlta53LTW2tvb+4phAQEB+ty5c/q9997T//3f/6211rq4uFgPHDhQHzt2TM+dO1e/+uqrtr8hNzdXnz9/XoeHh+tjx45V+xuff/55/cknn2ittb506ZKOjIzU+fn5dS5bezVVqvabtgKSdAMytiFN/rYplQ2KvbsuleMXCogI9q7/Q0I0kYiICGJjYwGjlcYxY8aglCI2NrbWpmTvvPNOTCYTUVFRdpsBtueNN97gnnvusT0EZNOmTQwbNow1a9bw+uuvU1hYyMWLF4mOjub222+v9tnDhw8TFhbGoEGDAPDz87ONGzNmDP7+/gBERUVx8uRJunbtWmsdH330EX/729/IysqytdXSvXt3hgwZYpvmiy++YMGCBZSXl3P27FkOHDhARUUFERERREZGAvDQQw/ZHsxR1erVq6s91i83N9e2lzFhwgQ8PDzw8PAgNDS0wcuuJm29AGPlypXs2bPHtveUk5PDkSNHGDRoENOnT6esrIw777yT+Ph41q5dy4gRI4iIiAAuN828cuVKli9fbmsTqLi4mFOnTgFXv2wby+nCHYwGxT7YcJz3NxzjT3fFOroc4Qi3XtsJucaqbDoWwGQy2d6bTKZam5Kt+pnKoKlq2rRp7Nq1i86dO1/x2DcfHx9GjRrFTz/9REJCAk8++SRJSUl07dqVl19++ZqbFQb7zd/27t2bU6dOkZeXh6+vL9OmTWPatGnExMTYWpWs2qzw8ePHmTt3Ltu3b6dDhw5MnTrVVlN9zQoDVFRUsHnzZlsb9FdTa0McO3YMFxcXQkND0Vozb948xo0bd8V069ev57vvvuPhhx9mzpw5BAQE2K1fa82yZcvo27dvteFbt25tknqvhlMdc68U6mvm7oFdWLojjcy8EkeXI0SjfPTRR7U+z7O8vJytW7fSq1cvW2gGBweTn59f7YRs1eZr+/XrR3p6Otu3bwcgLy+vwUHj5eXFo48+yuzZs23zs1gslJaW2p0+NzcXb29v/P39ycjIYMWKFbYajh8/TmqqcfFDbe3Cjx07ttqTkOy16V7V1TTTm5mZyaxZs5g9ezZKKcaNG8e7775rOzmckpJCQUEBJ0+eJDQ0lMcee4xHH32UnTt3MnToUNatW2c7kV3ZNPO4ceOYN2+ebSW9a9eueutwc3Oze0K6sZwy3EEaFBPObc6cOcTHx9O/f39iY2OZNGkSAQEBPPbYY8TGxnLnnXfaDruAceJ21qxZxMfHY7FYWLJkCb/4xS+Ii4vjlltusbuFX5s//vGPhIWFERMTw4ABA7jxxhuZMmUKnTt3vmLauLg4BgwYQHR0NNOnT7c9uclsNrNgwQImTJjADTfcQPfu3e3O66233iIpKYn+/fsTFRXF/Pnz66ytZhPKNVWevIyOjubmm29m7NixtpO8M2bMICoqioSEBGJiYnj88ccpLy9n7dq1xMfHM2DAAJYtW8bTTz9NSEgICxYsYNKkScTFxXH//fcD8Pvf/56ysjL69+9PTEwMv//97+tdnjNnzqR///5NfkK1zTf5W5dZn+xgU+oFNj8/Bm8PpzwCJaqQJn+Fs2nXTf7WpbJBsX9sO+XoUoQQokU5dbjbGhT76bg0KCaEaFecOtzBaFDsrDQo1m446jCjEE2tsb9lpw93aVCs/TCbzWRlZcm/s2jztNZkZWXZ7gK+Fk5/ltFkUjw+ohe//nI3a1MyGd031NEliWYSHh5OWloamZnScJxo+8xmM+Hh4df8eacPd4Db4zozd+Vh3luXKuHuxNzc3Gx3DArR3jn9YRmQBsWEEO1Puwh3gAcGd8PX7CrNAQsh2oV2E+6VDYr9sP8cxy8UOLocIYRoVu0m3MFoUMzNxcT7G445uhQhhGhW7SrcQ33N3J0QztIdaZzPa3hbGkII0da0q3AHeHxETywVmnfXyrF3IYTzanfh3iPYm7sTuvDpllOkZxc5uhwhhGgW7S7cAX5xUyQazdtrjjq6FCGEaBbtMty7BnrxwKBufLH9NKeyCh1djhBCNLl2Ge4As2/qjYtJ8daPRxxdihBCNLl2G+4d/cw8NKQ7X+1MIzUz39HlCCFEk2q34Q7wxKheeLi68L+rZetdCOFc6g13pVRXpdQapdRBpdR+pdTTdqZRSqm3lFJHlVJ7lFIJzVNu0wr28WDa8B78a086h87lOrocIYRoMg3Zci8Hfq21vg4YAjyllIqqMc2tQKS1mwm826RVNqOZI3ri4+7KX1elOLoUIYRoMvWGu9b6rNZ6p7U/DzgIdKkx2R3Ax9qwBQhQSoU1ebXNIMDLnUdvjODf+zPYm5bj6HKEEKJJXNUxd6VUD2AAsLXGqC7A6Srv07hyBdBqTb8hggAvN/6y6rCjSxFCiCbR4HBXSvkAy4Bfaa1rHqBWdj5yxbPOlFIzlVJJSqmk1vS0HD+zG4+P6MXaw5nsOHnR0eUIIUSjNSjclVJuGMH+qdb6KzuTpAFdq7wPB654IrXWeoHWOlFrnRgSEnIt9TabKcO6E+zjzl9WyrF3IUTb15CrZRTwIXBQa/1mLZMtBx6xXjUzBMjRWp9twjqbnZe7K0+M6s2m1Cw2Hb3g6HKEEKJRGrLlPhx4GLhJKZVs7X6mlJqllJplneZ74BhwFHgfeLJ5ym1ek6/vRic/M39ZlYLWVxxVEkKINqPeB2RrrX/C/jH1qtNo4KmmKspRzG4uzL6pNy98s491KZmMkodpCyHaqHZ9h6o99yV2JbyDJ39ZKVvvQoi2S8K9BndXE78cE8neMzmsPJDh6HKEEOKaSLjbMWlAFyKCvfnrqhQqKmTrXQjR9ki42+HqYuJXN0dy6Fwe3+1tUxf9CCEEIOFeq9v6d6ZPRx/+ujqFckuFo8sRQoirIuFeCxeT4v/d0odjmQV8k3zF/VhCCNGqSbjXYVx0J6I7+/G3/6RQJlvvQog2RMK9Dkopfj22D6cvFvFlUpqjyxFCiAaTcK/H6L6hDOgWwLwfj1BcZnF0OUII0SAS7vVQSvHM2L6czSnmH9tOObocIYRoEAn3BhjWK4ghPQP5vzWpFJXK1rsQovWTcG8A49h7Xy7kl/Dx5hOOLkcIIeol4d5Ag3oEMqJPCPPXpZJfUu7ocoQQok4S7lfh17f04VJhGR/9dNzRpQghRJ0k3K9CXNcAbr6uIws2HONsTpGjyxFCiFpJuF+l34zvi9Zw9zubOHo+z9HlCCGEXRLuV6lPR18+nzmEUovmnvmb2XnqkqNLEkKIK0i4X4OYLv589cQw/D3d+Pn7W1hz6LyjSxJCiGok3K9RtyAvls4aRu9QH2Z8nMTSHdI8gRCi9ZBwb4QQXw8+nzmUIT0DeebL3cxflyqP5hNCtAoS7o3k4+HKwqmDuK1/GH9ecYhXvzsoT28SQjicq6MLcAYeri689cAAgn08+PCn41zIL+GNe+Jwd5V1pxDCMSTcm4jJpHjp9ihC/Tx4/YfDXCwo5d2HBuLjIYtYCNHyZNOyCSmleHJUb16/uz+bUrP4+ftbuJBf4uiyhBDtkIR7M7hvUFfee2ggh8/lcc+7mzh9sdDRJQkh2hkJ92Zyc1RHPnvsei4VljHp3U0cSM91dElCiHZEwr0ZDeweyNJZQ3E1Ke5/bzObU7McXZIQop2QcG9mkR19WfbEMDr6m5mycBsr9p51dElCiHZAwr0FdA7wZOmsocR08ePJz3byyZaTji5JCOHkJNxbSICXO5/OGMLovqH8/pt9EvBCiGYl4d6CPN1deO/hgYzpF8qL3+7jh31yiEYI0Twk3FuYm4uJt3+eQHzXAH75eTLbjl90dElCCCck4e4Anu4ufDhlEOEBnsxYvJ2UDHnohxCiaUm4O0igtzuLpw/Gw82FKQu3kZ4tj+0TQjQdCXcH6hroxeJpg8kvLmfKwm3kFJY5uiQhhJOQcHewqM5+vPfwQE5kFfDYx0kUl1kcXZIQwgnUG+5KqYVKqfNKqX21jB+llMpRSiVbuxebvkznNqx3MG/eF8+2Exd5+vNdWKQ9eCFEIzVky30RML6eaTZoreOt3SuNL6v9uT2uMy/eFsW/92fw0vJ98kQnIUSj1NvYuNZ6vVKqR/OXIqbfEEFGbjHvrT9GJz8zs2+KdHRJQog2qqmOuQ9VSu1WSq1QSkXXNpFSaqZSKkkplZSZmdlEs3Yuz47vx10DujB3ZQpfbD/t6HKEEG1UU4T7TqC71joOmAd8U9uEWusFWutErXViSEhIE8za+ZhMitfu7s+NkcE8//VefjyU4eiShBBtUKPDXWudq7XOt/Z/D7gppYIbXVk75u5q4t2HBhIV5seTn+5k16lLji5JCNHGNDrclVKdlFLK2j/Y+p3ScHkj+Xi4snDqIEJ9zUxftJ3UzHxHlySEaEMacinkP4DNQF+lVJpS6lGl1Cyl1CzrJPcA+5RSu4G3gAe0XOrRJEJ8Pfh4+mBMSjFl4TbO5xY7uiQhRBuhHJXDiYmJOikpySHzbmt2n87mwfe30D3ImyWPD8HP7ObokoQQDqKU2qG1TqxvOrlDtQ2I6xrAO5MTOJKRx6xPdlBSLnexCiHqJuHeRozqG8prd/dnU2oWz3y5hwq5i1UIUYd6b2ISrcfdA8M5n1fCaz8cItjHnRdvi8J6LlsIIaqRcG9jZo3syfm8Yj7aeIIQXw+eHNXb0SUJIVohCfc2RinF7ydEkZVfyus/HCbYx4P7Ers6uiwhRCsj4d4GmUyKuffGcamwlOe/2kuglzs3R3V0dFlCiFZETqi2Ue6uJuY/NJCYzn489dlOkk7Is1iFEJdJuLdh3ta7WLsEeDJ90XYOn5NnsQohDBLubVyQjweLpw/G7ObCIwu3knap0NElCSFaAQl3J9A10IvF0wdTWGrhkYXbuFhQ6uiShBAOJuHuJK4L8+PDKYNIu1TEtEXbKSwtd3RJQggHknB3IoMjApn34AD2pmXzxN93UmapcHRJQggHkXB3MuOiO/Gnu2JZl5LJb5ZKMwVCtFdynbsTemBwNy7klzB3ZQpB3u78bsJ10kyBEO2MhLuTemp0bzLzSvjgp+OE+Hrw+Mheji5JCNGCJNydlFKKl26P5kJBKf+z4hBBPh7cMzDc0WUJIVqIhLsTM5kUb94XR3ZhKc8u20Ogtxs39ZNmCoRoD+SEqpPzcHXhvYcTbQ/b3nFSmikQoj2QcG8HfDxc+WjaIDr5mZm+KIkjGdJMgRDOTsK9nQj28eCTR6/H3dXEIwu3yRa8EE5Owr0d6RroxeJpg6nQmrvf3cxTn+3k9EVpi0YIZyTh3s5EdfZjzTOjeHpMJP85mMGYN9fx2g+HyCsuc3RpQogmJOHeDnm5u/Jft/RhzTOjuC02jHfXpjJ67lo+23oKi9zRKoRTkHBvx8L8PXnz/ni+fWo4EcHe/PbrvUx4awM/Hbng6NKEEI0k4S6I6xrAF48P5Z3JCRSUlvPQh1t5dNF2jp7Pd3RpQohrJOEuAOOO1p/FhrHqv0by/K392Hb8IuP/dz0vL9/PJWkfXog2R8JdVGN2c+Hxkb1YM2cU9w/qysebTzDyjTV8sOEYpeXShLAQbYWEu7Ar2MeDP94Vy4qnRxDXNYBXvzvIuP9dz8r959BaTroK0dpJuIs69e3ky8fTB/PRtEG4mBQzP9nBz9/fyoH0XEeXJoSog4S7qJdSitF9Q1nx9I389x3RHDqXy23zNvDCN3vlea1CtFIS7qLB3FxMPDy0B2ueGcUjQ3vwj22nGT13LYs2HqdcHuknRKsi4S6uWoCXOy9PjOb7X95ITBc/Xv7nAX721gY2HpXr44VoLSTcxTXr28mXvz96Pe89PJCiMguTP9jK458kSXs1QrQCEu6iUZRSjIvuxKr/GsmccX1Zn3KBMW+uY+6/D1NYWu7o8oRotyTcRZMwu7nw1OjerHlmFD+L6cTba45y09x1fLPrjFw6KYQD1BvuSqmFSqnzSql9tYxXSqm3lFJHlVJ7lFIJTV+maCs6+Zv53wcGsOyJoYT4evCrJcncM38ze9NyHF2aEO1KQ7bcFwHj6xh/KxBp7WYC7za+LNHWDeweyLdPDee1u2M5mVXAxP/7iWeX7uFCfomjSxOiXag33LXW64G6HttzB/CxNmwRNCA5AAAVP0lEQVQBApRSYU1VoGi7TCbF/YO68eMzo3h0eATLdqYx+o21fLDhGMVlFkeXJ4RTa4pj7l2A01Xep1mHCQGAn9mNF26L4odfjSChewde/e4gN7y2hrd/PEJ2odwEJURzaIpwV3aG2T2DppSaqZRKUkolZWZmNsGsRVvSO9SHRdMG8dlj1xPd2Y+5K1MY9ucfeXn5frl8Uogm5toE35EGdK3yPhxItzeh1noBsAAgMTFRLqFoh5RSDOsVzLBewRw6l8uC9cf4+5aTfLLlJBNiw5g5oicxXfwdXaYQbV5TbLkvBx6xXjUzBMjRWp9tgu8VTq5fJz/evC+e9b8ZzfThPfjx0Hlum/cTkz/YwrqUTLmEUohGUPX9B1JK/QMYBQQDGcBLgBuA1nq+UkoBb2NcUVMITNNaJ9U348TERJ2UVO9koh3JLS7js62n+GjjcTJyS+jXyZfHR/bktv6dcXORWzKEAFBK7dBaJ9Y7naO2jiTcRW1Kyi18m5zO++uPceR8Pp39zUy/IYIHBnfDx6MpjiQK0XZJuIs2r6JCszblPO+tO8bW4xfxNbsy+fruTBveg45+ZkeXJ4RDSLgLp7L7dDYL1h9jxb6zuJgUt0R15N6BXbkxMhhXOWQj2hEJd+GUTmYVsGjTCb5NTudiQSmhvh7cldCFeweG0zvU19HlCdHsJNyFUystr+DHQ+dZuuM0aw5nYqnQxHcN4N7EcG7r3xl/TzdHlyhEs5BwF+1GZl4J3+w6w5c7TpOSkY+Hq4lx0Z24NzGcYb2CcTHZu89OiLZJwl20O1pr9p7J4cukNL5NPkNucTlh/mbuTgjn7oHhRAR7O7pEIRpNwl20a8VlFlYfzGDpjjTWp2RSoWFQjw7cO7ArP+sfJpdUijZLwl0Iq3M5xXxtPWxzLLMATzcXhvcOYkSfEEb2CaF7kGzRi7ZDwl2IGrTW7DqdzTe7zrD2cCanrI2V9QjysgX9kJ5BeMtWvWjFGhru8isW7YZSioRuHUjo1gGAExcKWJeSybqUTL5MSuPjzSdxdzGR2KMDI/uEMLJvCH07+mK0sCFE2yJb7kJgNHmQdOIS61IyWZ+SyaFzeQB09PNgRKQR9Df0DibAy93BlYr2Tg7LCNEI53KKWZ+Sybojmfx05AI5RWWYFMR1DeDGyBB6hXjT0c9s7TzwcpedYNEyJNyFaCLllgp2p+UYYZ+Sye60bGr+t/E1u9LRz0wnPzOhfh50sgW/Ef4d/cyE+HpI65ai0STchWgmBSXlnM0pJiPX6M7lFnM+t4RzOcVk5BWTkVPM+bwSyiuq/99SCoK8Pejk70GYvyed/c10DvC0dkZ/qK9ZbroSdZITqkI0E28PV3qH+tA71KfWaSoqNFkFpbYVQEZuiXUlYKwMTmUVsiU1i7yS8mqfczEpOvmZ6RxgNlYAAZ50qdLfOcCMv6cbSim01pRaKiguq6CkzGK8lhuvxeUWSsoqKC6zUFxefVxpeQWB3m7VvlMOKzkf+RcVohmYTIoQXw9CfD3qfGxgbnEZZ7OLSc8pIj3b6M5mF3Mmu4jk09ms2HeWMkv1PQCzm3Fop6S84orDQ9fK39ONMH8zXQI8CbOuTLoEeBJm3bvo6GfG3VUOKbUlEu5COJCf2Q2/Tm707WS/RcuKCs2F/BLSc4pt4X8upxiTSWF2NeHh5oKHqwmzmwvmKv2Xh5nwcDVeK4e7uZi4WFBqfF9OEenZxZzNubxSSTp5iZyismp1KAUhPh6EWfck+nT0pX+4P7FdAgjx9WiJRSWukoS7EK2YyaQI9TMT6mcmvmtAk32vt4crXQO9ah1feV4hPbuIs1VWAOnZxRw8m8eKfedsew1h/mZiu/gbXbjxGuQjge9oEu5CiCvUd14hv6ScA+m57EnLZu+ZHPam5bDyQIZtfJcAT/qH+xPTxd+6he8v9wi0MAl3IcRV8/FwZXBEIIMjAm3DcovL2H8ml71nstmTlsO+Mzms2HfONr5boJdt674y8H3N0u5+c5FwF0I0CT+zG0N7BTG0V5BtWE5hGfvSc2xhv+dMNt/tPQsYx/F7h/gQ1zWAuK4BxIcH0LeTr5y4bSIS7kKIZuPv5cbw3sEM7x1sG3apoJTdadnsPp3D7rRs1hw6z9IdaQC4u5qI7uxHXHgA8dbQ7xHkJe37XAO5iUkI4VBaa9IuFVkD3wj9vWdyKCqzAMZlmv3D/Y2wDzcCvz1foSM3MQkh2gSlFF0Dvega6MVt/TsDRpMPR87nG2Gflk3y6RzeWZuKxXrXb89gb4b1DmJ4r2CG9gqSk7V2yJa7EKJNKCwtZ396LrtOXWJzahbbjl+koNSCUhDd2Y/hvYIZ1juYwT0C8XR3cXS5zUbalhFCOLUySwW7T2ez8WgWG1MvsOvUJcosGjcXxYBuHbihdzDDewfRPzzAqRpsk3AXQrQrhaXlbD9xiU1HL7Ax9QL703PRGrzdXbi+ZxDDegUxvHcwfTv6YmrDjbPJMXchRLvi5e5qPEGrTwhgXJWz5ZixVb/paBY/HjoPQJC3O7Hh/vTp6GvtjJu1nK3xNOf6a4QQwqqDtzu3xoZxa2wYAOnZRWxKzWJT6gUOns1jU2oWpeUVgHHNfXgHT/qE+tKnkxH4kaG+9A71wezWNo/fy2EZIUS7VG6p4OTFQo5k5JGSkU9KRh4pGXkcv1Bga4nTpKB7kDeRoT706ehLZEcf+nbyJSLYGw9Xx4S+HJYRQog6uLqY6BXiQ68QH8bHXB5eZqngxIUCUjLyOZyRZw3/PP5z6LztUkyTMppT6B3qQ69QH3qHWF9DffBrJU0qSLgLIUQVbi4mIjv6EtnRlwmE2YaXlFs4lllASkYeqefzOZqZz9Hz+axPuUCppcI2Xaivh63Rtd6hxsqjd6gPob4eLXqnrYS7EEI0gIerC9eF+XFdmF+14eWWCk5fKuLo+Xxbl5qZz9c7z1R70pav2dUW9OOjO3FzVMdmrbfthXtpAWz/EIY+Baa2eaJDCOE8XF1MRAR7ExHszS1VAltrzfm8kitCf31KJt0CvSTcr7D/G1j1ezi5Ee7+ADzsP8FGCCEcSSlFRz8zHf3M1RpOA+MJW82t7d22NWAyTPgLHFkFC2+FnDRHVySEEFelJW6ialC4K6XGK6UOK6WOKqWeszN+qlIqUymVbO1mNH2pVQyaAZO/gOyT8P4YSN/VrLMTQoi2pt5wV0q5AP8H3ApEAQ8qpaLsTLpEax1v7T5o4jqv1PtmeHQluLgbW/AH/9nssxRCiLaiIVvug4GjWutjWutS4HPgjuYtq4FCr4PH/gOdYmDJw7Dxb+Cgm7KEEKI1aUi4dwFOV3mfZh1W091KqT1KqaVKqa5NUl1D+ITClH9C9F2w6kX45y/BUtZisxdCiNaoIeFu78h/zc3jfwI9tNb9gdXAYrtfpNRMpVSSUiopMzPz6iqti5sn3P0hjJgDOz+Gv0+CoktN9/1CCNHGNCTc04CqW+LhQHrVCbTWWVrrEuvb94GB9r5Ia71Aa52otU4MCQm5lnprZzLBTS/AnfPh5Gb44Ba4eKxp5yGEEG1EQ8J9OxCplIpQSrkDDwDLq06glAqr8nYicLDpSrxK8Q/CI99C4QX44GY4tcVhpQghhKPUG+5a63JgNvBvjND+Qmu9Xyn1ilJqonWyXyql9iuldgO/BKY2V8EN0mM4zPgPeHaAxbfDni8cWo4QQrQ0527yt/AifPEInNgAI5+DUc8ZDTcLIUQb1dAmf9veHapXwysQHvoKBjwE6/4My2ZAWbGjqxJCiGbX9tqWuVqu7jDxbQjqDatfhpzT8MBn4B1c70eFEKKtcv5wB+NQzA3/BYE94avHYf4N0GUgeAWBd4i1C7Z2IeAVbIxzaR+LRwjhfNpXekXdAf7hsOZPcPE4nN4GhVmgLfan9+xwOfyrrgi8gozWKN29rZ2P8erhc7nf1dy0x/e1Nm7OspRCRbkxf2nyWAhRi/YV7mBssT+07PL7igoozoaCTCi4YH3NNEK/sr8gCzIPG80MF17kynu47FAma9D7XLkScPM0AtpSCuUlxqulFMpLwVJSvb/cOs5SUnMGYPY3zit4BhorIi/rq2dglf4O1cd5+Nlf6WhtrakMKsqgwnK532J9X9mvLcbf4uFrdG5ecqJaiFam/YV7TSaTEXxegRDSt/7pLeXG3a+l+caDQ0rzq/QXQEmN96V5VfoLIO8slBWBi5vR6Jmrh/Hq5nW538XdOFfg4mEd5mbttw5TJijJNeoovAhFF43r+i+kQFE2lOTU8fe6GgGPNv6WqoF9rZTJGvR+NV6rdjWGVa7k3LysneflYa6exr+LEOKaSbhfLRdX8AkBmvgO26ZkKYPinMvBb1sJXDLeF+cYgWxyNToXNzC5GX+bydXa71b7eOUCZYXGCqYkD4qtryV5l4cVZsGlE5eHlxVc3d/g6gnuVYK/ciVQOczsD54BYA6w7p3Y6Tf7y6Er0W5JuDsjF7fLJ4hbC0u5sUdTuQIoKzJWEKWFxmtZoTGstODyOHvDii4ZD2gpzjH2UsqL6pipArNfHSsAf+v7KiuD5lgxVFis9Vv/hvLiyytXF3frnlnlCtT6Xg5ziUaScBctw8XVCE7PgKb93rJi45xJUbYR/PX155y53F9RT+uhHpUrhqorAX/w8DfOT5QVVA/taq9VVkpXnC9pAOViJ/jdqhzOM1v3aDyr7N1U6Xf1tDO+yquHr3XF5986zplYyu0sw8JahhWDT0cI6mV0Zn/H1t5KSbiLts3NDG6dwLfT1X1Oa+ueQLaxF1AZ+LZXO8MuHDX6i3ONkLWdK6hy+MgryE6oelcf5uph1GAprX4FlO19lRPZtv5SIwBtJ99LjPqLcyEv43L4lVtXLJbShi8L5WLdi/G/HPiVK7bK97Zh/saKQVfZGykvrtFvDeDKWmqdpsreW30r2rp4hxr3sQT1sr72huBI6NDj8rJuhyTcRfuk1OWrmPztPZ6gjbOUVwnXGlu9ledLinOMlUNxTpX31mH5qZeHleZf3bxNbpdXZq5m60rObOxNmP2NlXF9K0C7r9aT7i7uxoUJWUerdKmQ8oNxdVslZYKAbpcDv3IFENjLWEGBdY9FVe9vyKvJ1fF7O/WQcBfCGbm4govv5RBrDEu59UR5lRWCi1v1Q0OunpcDvCVu/vMMMJ7EVlNRNlxMNcK+avif2nL1K6m6uHhcvgzZq+qlyIF2Lkm2vpoDWvTGSAl3IUTdXFwvXy7c2nkGGPeydKnxSAmtIT/j8lZ+WRGgqzyWs7K/ga8ledZLkK1XoGUevnxlWkV57fWZ/Y2wHzQDhs1uhgVwmYS7EML5KWWcl/HtBD1uaL75aG3nHhTrCsD2/qJxQriZSbgLIURTUeryCegOPRxaitwGKIQQTkjCXQghnJCEuxBCOCEJdyGEcEIS7kII4YQk3IUQwglJuAshhBOScBdCCCektG7AI+OaY8ZKZQInr/HjwcCFJiynqbX2+qD11yj1NY7U1zitub7uWut6nxbksHBvDKVUktY60dF11Ka11wetv0apr3GkvsZp7fU1hByWEUIIJyThLoQQTqithvsCRxdQj9ZeH7T+GqW+xpH6Gqe111evNnnMXQghRN3a6pa7EEKIOrTqcFdKjVdKHVZKHVVKPWdnvIdSaol1/FalVI8WrK2rUmqNUuqgUmq/UuppO9OMUkrlKKWSrd2LLVWfdf4nlFJ7rfNOsjNeKaXesi6/PUqphBasrW+V5ZKslMpVSv2qxjQtvvyUUguVUueVUvuqDAtUSq1SSh2xvnao5bNTrNMcUUpNacH63lBKHbL+G36tlAqo5bN1/h6asb6XlVJnqvw7/qyWz9b5/70Z61tSpbYTSqnkWj7b7MuvSWmtW2UHuACpQE/AHdgNRNWY5klgvrX/AWBJC9YXBiRY+32BFDv1jQL+5cBleAIIrmP8z4AVGE8IHgJsdeC/9TmM63cduvyAEUACsK/KsNeB56z9zwGv2flcIHDM+trB2t+hheobC7ha+1+zV19Dfg/NWN/LwDMN+A3U+f+9ueqrMf4vwIuOWn5N2bXmLffBwFGt9TGtdSnwOXBHjWnuABZb+5cCY5RqmUeSa63Paq13WvvzgINAl5aYdxO6A/hYG7YAAUqpMAfUMQZI1Vpf601tTUZrvR64WGNw1d/ZYuBOOx8dB6zSWl/UWl8CVgHjW6I+rfVKrXXlgzu3AOFNPd+GqmX5NURD/r83Wl31WbPjPuAfTT1fR2jN4d4FOF3lfRpXhqdtGuuPOwcIapHqqrAeDhoAbLUzeqhSardSaoVSKrpFCwMNrFRK7VBKzbQzviHLuCU8QO3/oRy5/Cp11FqfBWOlDoTamaa1LMvpGHtj9tT3e2hOs62HjRbWclirNSy/G4EMrfWRWsY7cvldtdYc7va2wGte2tOQaZqVUsoHWAb8SmudW2P0ToxDDXHAPOCblqwNGK61TgBuBZ5SSo2oMb41LD93YCLwpZ3Rjl5+V6M1LMvfAeXAp7VMUt/vobm8C/QC4oGzGIc+anL48gMepO6tdkctv2vSmsM9Deha5X04kF7bNEopV8Cfa9slvCZKKTeMYP9Ua/1VzfFa61ytdb61/3vATSkV3FL1aa3Tra/nga8xdn2rasgybm63Aju11hk1Rzh6+VWRUXm4yvp63s40Dl2W1hO4twGTtfUAcU0N+D00C611htbaorWuAN6vZb6OXn6uwCRgSW3TOGr5XavWHO7bgUilVIR16+4BYHmNaZYDlVcl3AP8WNsPu6lZj899CBzUWr9ZyzSdKs8BKKUGYyzvrBaqz1sp5VvZj3HSbV+NyZYDj1ivmhkC5FQefmhBtW4tOXL51VD1dzYF+NbONP8GxiqlOlgPO4y1Dmt2SqnxwLPARK11YS3TNOT30Fz1VT2Pc1ct823I//fmdDNwSGudZm+kI5ffNXP0Gd26OoyrOVIwzqL/zjrsFYwfMYAZY3f+KLAN6NmCtd2Asdu4B0i2dj8DZgGzrNPMBvZjnPnfAgxrwfp6Wue721pD5fKrWp8C/s+6fPcCiS387+uFEdb+VYY5dPlhrGjOAmUYW5OPYpzH+Q9wxPoaaJ02EfigymenW3+LR4FpLVjfUYzj1ZW/w8oryDoD39f1e2ih+j6x/r72YAR2WM36rO+v+P/eEvVZhy+q/N1VmbbFl19TdnKHqhBCOKHWfFhGCCHENZJwF0IIJyThLoQQTkjCXQghnJCEuxBCOCEJdyGEcEIS7kII4YQk3IUQwgn9fxzao81KZrCCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Losses\n",
    "plt.plot(loss_history_bgd, label=\"Batch Gradient Descent\")\n",
    "plt.plot(loss_history_mbgd, label=\"mini-Batch Gradient Descent\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Gradient Descent\n",
      "Loss: 0.538221\n",
      "Accuracy: 86.68%\n",
      "\n",
      "mini-Batch Gradient Descent\n",
      "Loss: 0.296906\n",
      "Accuracy: 91.41%\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rusty\\Desktop\\mnist_experiments\\activation.py:9: RuntimeWarning: overflow encountered in exp\n",
      "  return 1/(1+np.exp(-x))\n"
     ]
    }
   ],
   "source": [
    "def test(test_x, test_y, w1, b1, w2, b2):\n",
    "    # Flatten input (batch_size, 28, 28) -> (batch_size, 784) \n",
    "    x = test_x.reshape(test_x.shape[0], -1)\n",
    "    m = x.shape[0]\n",
    "    \n",
    "    # Turn labels into their one-hot representations\n",
    "    y = one_hot_encoder(test_y)\n",
    "\n",
    "    # Forward Pass\n",
    "    h1 = np.dot(x, w1) + b1\n",
    "    a1 = sigmoid(h1)\n",
    "    h2 = np.dot(a1, w2) + b2\n",
    "    a2 = softmax(h2)\n",
    "    out = a2\n",
    "\n",
    "    # Cross Entropy Loss\n",
    "    loss = cross_entropy_loss(out, test_y)\n",
    "    print(\"Loss: {:.6f}\".format(loss/m))\n",
    "\n",
    "    # Compute and print accuracy\n",
    "    pred = np.argmax(out, axis=1)\n",
    "    pred = pred.reshape(pred.shape[0], 1)\n",
    "    acc = np.mean(pred == test_y)\n",
    "    print(\"Accuracy: {:.2f}%\\n\".format(acc*100))\n",
    "    \n",
    "print(\"Batch Gradient Descent\")\n",
    "test(test_x, test_y, w1_bgd, b1_bgd, w2_bgd, b2_bgd)\n",
    "print(\"mini-Batch Gradient Descent\")\n",
    "test(test_x, test_y, w1_mbgd, b1_mbgd, w2_mbgd, b2_mbgd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}