04 Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regularization: Dropout, and Batch Normalization\n",
    "\n",
    "For educational purposes and for the reason of exploring neural network concepts with the least amount of, we will move away from using more complex optimization algorithms, and stick with the good ol' vanilla SGD. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified data/t10k-images-idx3-ubyte.gz\n",
      "Found and verified data/t10k-labels-idx1-ubyte.gz\n",
      "Found and verified data/train-images-idx3-ubyte.gz\n",
      "Found and verified data/train-labels-idx1-ubyte.gz\n",
      "Found and verified data/t10k-images-idx3-ubyte.gz\n",
      "Found and verified data/t10k-labels-idx1-ubyte.gz\n",
      "Found and verified data/train-images-idx3-ubyte.gz\n",
      "Found and verified data/train-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import mnist\n",
    "from activation import relu, sigmoid, sigmoid_prime, softmax\n",
    "from helper import one_hot_encoder\n",
    "from initializer import initialize_weight\n",
    "%matplotlib inline\n",
    "\n",
    "from utils import dataloader\n",
    "from losses import cross_entropy_loss\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Load Dataset\n",
    "train_x, train_y = mnist.load_dataset(download=True, train=True)\n",
    "test_x, test_y = mnist.load_dataset(download=True, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropout: Neural Networks as ensamble of simpler neural networks\n",
    "\n",
    "The idea of dropout is to train simpler sub-networks of the whole network, while also sharing parameters with the other sub-networks. You can also think of dropout as convolution with very weak spatial prior. Implementing dropout is as easy as: \n",
    "\n",
    "### Forward\n",
    "1. Generate a dropout mask. Dropout mask is basically a __Bernoulli Distribution__ (parameters with binary values) with shape same as activation output's shape, parametrized by the keep probability \n",
    "    \n",
    "    `mask = np.random.Binomial(1, keep_prob, size=activation_output.shape)`\n",
    "\n",
    "2. Apply/Multiply the dropout mask \n",
    "\n",
    "    `output *= mask`\n",
    "    \n",
    "### Backprop\n",
    "1. Reuse the mask from forward prop, and apply to the gradient with respect to outputs\n",
    "\n",
    "    `d_output *= mask`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_x, train_y, learning_rate=0.1, num_epochs=50, batch_size=1, drop_prob=0):\n",
    "    # Flatten input (num_samples, 28, 28) -> (num_samples, 784) \n",
    "    x = train_x.reshape(train_x.shape[0], -1)\n",
    "    num_samples = x.shape[0]\n",
    "    \n",
    "    # Turn labels into their one-hot representations\n",
    "    y = one_hot_encoder(train_y)\n",
    "\n",
    "    # Make a data loader\n",
    "    trainloader = dataloader(x, y, batch_size=batch_size, shuffle=True)\n",
    "    \n",
    "    # Initialize weights\n",
    "    w1, b1 = initialize_weight((784, 256), bias=True)\n",
    "    w2, b2 = initialize_weight((256, 10), bias=True)\n",
    "\n",
    "    loss_history = []\n",
    "    for epoch in range(1, num_epochs+1):\n",
    "        print(\"Epoch {}/{}\\n===============\".format(epoch, num_epochs))\n",
    "\n",
    "        batch_loss = 0\n",
    "        acc = 0\n",
    "        for inputs, labels in trainloader:\n",
    "            # Number of samples per batch\n",
    "            m = inputs.shape[0]\n",
    "            \n",
    "            # Forward Prop\n",
    "            h1 = np.dot(inputs, w1) + b1\n",
    "            a1 = sigmoid(h1)\n",
    "            \n",
    "            # Dropout Forward\n",
    "            if (drop_prob != 0):\n",
    "                # Bernoulli Distribution\n",
    "                keep_prob = 1 - drop_prob\n",
    "                mask1 = np.random.binomial(1, keep_prob, size=a1.shape) \n",
    "                \n",
    "                # Multiply mask\n",
    "                a1 *= mask1 \n",
    "            \n",
    "            h2 = np.dot(a1, w2) + b2\n",
    "            a2 = softmax(h2)\n",
    "            out = a2\n",
    "\n",
    "            # Cross Entropy Loss\n",
    "            batch_loss += cross_entropy_loss(out, labels.argmax(axis=1).reshape(m,1))\n",
    "\n",
    "            # Compute Accuracy\n",
    "            pred = np.argmax(out, axis=1)\n",
    "            pred = pred.reshape(pred.shape[0], 1)\n",
    "            acc += np.sum(pred == labels.argmax(axis=1).reshape(m,1))\n",
    "\n",
    "            # Backward Prop\n",
    "            dh2 = a2 - labels \n",
    "            dw2 = (1/m) * np.dot(a1.T, dh2)\n",
    "            db2 = (1/m) * np.sum(dh2, axis=0, keepdims=True)\n",
    "\n",
    "            # Dropout Backprop\n",
    "            da1 = np.dot(dh2, w2.T)\n",
    "            if (drop_prob != 0):\n",
    "                da1 *= mask1\n",
    "            \n",
    "            dh1 = da1 * sigmoid_prime(a1)\n",
    "            dw1 = (1/m) * np.dot(inputs.T, dh1)\n",
    "            db1 = (1/m) * np.sum(dh1, axis=0, keepdims=True)\n",
    "\n",
    "            # Weight (and bias) update\n",
    "            w1 -= learning_rate * dw1\n",
    "            b1 -= learning_rate * db1\n",
    "            w2 -= learning_rate * dw2\n",
    "            b2 -= learning_rate * db2\n",
    "            \n",
    "        loss_history.append(batch_loss/num_samples)\n",
    "        print(\"Loss: {:.6f}\".format(batch_loss/num_samples))\n",
    "        print(\"Accuracy: {:.2f}%\\n\".format(acc/num_samples*100))\n",
    "\n",
    "    return w1, b1, w2, b2, loss_history\n",
    "\n",
    "def cross_entropy_loss(out, y):\n",
    "    batch_size = y.shape[0]\n",
    "    y = y.reshape(batch_size)\n",
    "    log_likelihood = -np.log(out[np.arange(batch_size), y])\n",
    "    return np.sum(log_likelihood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "===============\n",
      "Loss: 0.860760\n",
      "Accuracy: 75.30%\n",
      "\n",
      "Epoch 2/20\n",
      "===============\n",
      "Loss: 0.621338\n",
      "Accuracy: 81.11%\n",
      "\n",
      "Epoch 3/20\n",
      "===============\n",
      "Loss: 0.598645\n",
      "Accuracy: 81.36%\n",
      "\n",
      "Epoch 4/20\n",
      "===============\n",
      "Loss: 0.565301\n",
      "Accuracy: 82.26%\n",
      "\n",
      "Epoch 5/20\n",
      "===============\n",
      "Loss: 0.546670\n",
      "Accuracy: 82.89%\n",
      "\n",
      "Epoch 6/20\n",
      "===============\n",
      "Loss: 0.521428\n",
      "Accuracy: 83.98%\n",
      "\n",
      "Epoch 7/20\n",
      "===============\n",
      "Loss: 0.493815\n",
      "Accuracy: 84.89%\n",
      "\n",
      "Epoch 8/20\n",
      "===============\n",
      "Loss: 0.503745\n",
      "Accuracy: 84.38%\n",
      "\n",
      "Epoch 9/20\n",
      "===============\n",
      "Loss: 0.491926\n",
      "Accuracy: 84.63%\n",
      "\n",
      "Epoch 10/20\n",
      "===============\n",
      "Loss: 0.487998\n",
      "Accuracy: 84.70%\n",
      "\n",
      "Epoch 11/20\n",
      "===============\n",
      "Loss: 0.457509\n",
      "Accuracy: 85.86%\n",
      "\n",
      "Epoch 12/20\n",
      "===============\n",
      "Loss: 0.448770\n",
      "Accuracy: 86.11%\n",
      "\n",
      "Epoch 13/20\n",
      "===============\n",
      "Loss: 0.448112\n",
      "Accuracy: 86.15%\n",
      "\n",
      "Epoch 14/20\n",
      "===============\n",
      "Loss: 0.444213\n",
      "Accuracy: 86.21%\n",
      "\n",
      "Epoch 15/20\n",
      "===============\n",
      "Loss: 0.441614\n",
      "Accuracy: 86.42%\n",
      "\n",
      "Epoch 16/20\n",
      "===============\n",
      "Loss: 0.436332\n",
      "Accuracy: 86.47%\n",
      "\n",
      "Epoch 17/20\n",
      "===============\n",
      "Loss: 0.431874\n",
      "Accuracy: 86.68%\n",
      "\n",
      "Epoch 18/20\n",
      "===============\n",
      "Loss: 0.433399\n",
      "Accuracy: 86.70%\n",
      "\n",
      "Epoch 19/20\n",
      "===============\n",
      "Loss: 0.422260\n",
      "Accuracy: 86.78%\n",
      "\n",
      "Epoch 20/20\n",
      "===============\n",
      "Loss: 0.416067\n",
      "Accuracy: 87.08%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Minibatch Gradient Descent with batch_size 64\n",
    "w1, b1, w2, b2, loss_history = train(train_x, train_y, learning_rate=0.1, num_epochs=20, batch_size=64, drop_prob=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABXUAAAD7CAYAAAAl6XdWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xe4FEXWx/FfAyooIEFQDKDCopgJxlck6BpQFAUVwZzjsgYUBR0GzAF1FUFcM4oBQTGhawBFzCgogiiuIIpKRpKkfv8YqK0qp4e5w525t+d+P8/js6fu6Z4ulqK6p2736SAMQwEAAAAAAAAA4qFSWXcAAAAAAAAAAJA9FnUBAAAAAAAAIEZY1AUAAAAAAACAGGFRFwAAAAAAAABihEVdAAAAAAAAAIgRFnUBAAAAAAAAIEYq1KJukAzqBcng2yAZVM1i262DZDAlSAabFaJvKL8YN8gF4wa5CJLBbkEy+CzLbY8NksEz+e4Tyj/GDUqKcxRywVyDXDDfICdBUE9B8K2CDY8bBcFeCoLxBegVyrmKON9UKesObIwgGSzxflRN0gNhIrwsYpdekh4NE+GKdftPltTIyleV9HqYCDuGifC3IBm8K+l8SfeVctdRRtb9g31A0mGS6kj6XtJ1YSJ8PcNu/ri5U9JxkraR9LOkm8NE+IQkMW6KV5AMLpV0pqQ9JQ0LE+GZG9jFHzcnSfqnpH0kfRImwrbrN2TcFK8gGdSR9LCkwyXNlXRtmAifzrBLf0l3ZrN/mAhHBcng5iAZ7BUmwkn5+jOg7ATJ4G+SvpI0PEyEp2bY1Bk3mfZn3BQnzlHYGMw1KIkgGTSTNFBSS0lzJPUME+HIDLv48812Sn0fay1pmaQbw0Q4WGK+KXpB0FVSQlJDSb9KOlNh+H7E1r0kPaowNW4UBI9J6iZppbXNlgrDNQrDSQqChQqCjgrDl/PWfxQc882GxfpO3TARVl//n6StJS2X9Hy6bdct5p0haai1/+7W/jUkzfT2f0rSBfnqP8pEFUk/SWojaUtJ10t6LkgGO6bbON24kbRUUsd1+58h6d4gGRxk5Rk3xekXSTdKemRDG0aMm/mS7pF0a8RujJviNFCpi8+tJXWXNChIBrun2zBIBg0ktZP0Ygn2H6bUhQiK00BJn2baIGLcbGh/xk3x4RyFjcFcg6wEyaCKpJckvaLUDTLnSxoaJIOmEdunm2+GSvqvUtc2R0u6OUgG7aw8800xCoK/S7pN0llKrb0cIumHiG3TjRtJul1hWN36b42VY9wUGeab7MR6UdfTRdLvkqJ+07O/pIVhIpwVkT9EUn1JL1g/+1jSzkEyaJR+F8RNmAiXhomwb5gIfwwT4dowEb6i1D/ylhG7/GXchIkwESbCqev2/1ipMXegtQ/jpgiFiXBEmAhflDQvi83TjZu3wkT4nFJfvNNh3BSZIBlsIamzpOvDRLgkTITjJI2SdFrELn+XNMH6zXI2+49R6gIFRSZIBl0lLZT09gY2dcZNlvuPEeOmqHCOQq6Ya1BCu0raVtLdYSJcEybCdyR9oOhrG2e+CZJBdUltJd0UJsJVYSKcKGm4pLOtfZhvilNSUj+F4UcKw7UKw58Vhj9HbLu/pIUKI9du0hkj6dB1C8IoDsw3WSimRd0zJD0RJsIwIr+npG83sP/wMBEuXf+DMBGuVurx/L1LrZcoV4JksLWkppImR2yScdwEyaCapH3t/Rk30Ibnm79g3BSlppLWhIlwmvWziZLS3qmrv46bbPafImnHIBnULIX+opxY9/fZT9KVWWz+l/kmi/0ZNxUb5yhIYq5BToKIn+0Rsb0/bgLvf/+yP/NNEQqCypJaSaqnIPheQTBLQXC/gqBaxB5R56mLFQTzFQSfKwg6O5nUAvEqSbuUZtdRpphvslAUi7pBMmio1OP0j2fYrJakPyL231ypO30fS5P+Y92+KDJBMthEqdvtHw8T4dSIzSLHzTqDlVpkecP7OeOmYtvQuInCuCku1SUt8n62SKlHztLxx002+6/fnnFTXPpLejhMhD9lsW26+WZD+zNuKjbOUViPuQYlNVWpp2N7BslgkyAZHK7U9/DNI7Z3xk2YCP9Q6k6764NkUDVIBi2UeirJ35/5prhsLWkTpdZcWitVv725pD4R26ebb/4l6W9KPV19vaTHFAT/523DuCkuzDdZKIpFXUmnSxoXJsL/ZthmgaK/SJ+gVC2xsWlyNZR6pAhFJEgGlSQ9qVStykszbBo5boJkcIdSv+U5Kc0d4oybii3TfJMJ46a4LJHk351UU9GLKf64yWb/9dszbopEkAz2UeplnndnuYszbrLcn3FTsXGOAnMNchImwlWSOilVVuNXpe7Sfk5S1GPy6eab7pJ2Uuo9J4OUusnG35/5prgsX/e/9ykMZysM50oaIKlDxPZ/HTdhOEFhOE9huFph+JpS4+YEbz/GTRFhvslOlbLuQCk5XdEvdlhvkqTLI3JpSzesK8zcRKk7MVEkgmQQKPU2+a0ldVg3WURJO26CZJCUdJSkNmEiXOzlGDfINN+kxbgpStMkVQmSwd/CRPjdup/trehyL5OUOh+VZP9mkn705yHEWltJO0qaGSQDKXXHduUgGewWJsIWabb3x002+zNuKjbOUZCYa5CjMBFOUupuOUlSkAzGK/qJ2b/MN2EinCHpGGv/pyV9YrWZb4pNGC5QEMySFFUq05fNeSqU/Vh9EGwraVOVsLwQyjfmmw2L/aJukAwOkrSdpOc3sOknkmoFyWC7MPG/gtxBMtheqTe5Xphmn/2UuhCZUVr9RbkwSKmLzMPCRLh8A9v+ZdwEyeBaSd0kHRImwnQvJGHcFKF1E34VSZWV+tJSVdLqdXV4fOnGTWWlHjuqIqnSuv3XWL9UYNwUmTARLg2SwQhJ/YJkcK5Sj5odJ+mgiF3+I+neIBlUDRPhiiz3byPp9fz9KVAGhkh6xmpfpdTCyUUR2zvjJsv9GTdFhnMUcsBcg5wEyWAvpX7xXEnSxZIaKH0ZQyn9fNNMqTvl/pR0kqTDlfputh7zTXF6VNJlCoLRStW+/aekVyK2/URSLQXBduZlakHQRdJoScuUekrgVEkdrX3aSnpHYfhnXnqPMsF8s2HFUH7hDEkj1tXLiBQmwpVK/eWf6qVOk/RhmAinp9mtu1I1U1Ek1r3V8AKlFkd+DZLBknX/dU+3fcS4uVlSQ0nfWftfZ+UZN8Wpj1KPDvVSajwsV0QdqIhxc9q6fQYpVUtquaSHrDzjpjhdLKmaUvWghkm6KEyEae/UDRPhb5LeUWrhNtv9T5H0YB76jTISJsJlYSL8df1/SpXhWBEmwjkR2zvjJsv9GTfFh3MUSoS5BhvhNEmzlbo2OVTS38NE+oW0iPnmCEk/KPWo9IWSjvTGDfNNceov6VOlFuimSPpC0k1ptwzTjpsekn5W6jH5OySdpzAcY+UZN8WJ+WYDgvAvpUCLV5AM6kl6X1LzDd2hGSSD+krV2G2+7rfRqKAYN8gF4wa5CJLBbko9UrRfmlrd/rYdJZ0WJsKTCtI5lFuMG5QU5yjkgrkGuWC+QU6C/40bhRt4ujYI9pQ0RGF4YCG6hvKrIs43FWpRFwAAAAAAAADirhjKLwAAAAAAAABAhcGiLgAAAAAAAADECIu6AAAAAAAAABAjLOoCAAAAAAAAQIxUKcnGQRDwVrXyY24YhvXKuhPZYNyUH2EYBmXdh2wwZsoV5hrkgnGDXDBukAvGDXLBuEEuGDcoMb6DIwdZzzXcqRtfM8q6AwAqBOYa5IJxg1wwbpALxg1ywbhBLhg3AAoh67mGRV0AAAAAAAAAiBEWdQEAAAAAAAAgRljUBQAAAAAAAIAYYVEXAAAAAAAAAGKERV0AAAAAAAAAiBEWdQEAAAAAAAAgRljUBQAAAAAAAIAYYVEXAAAAAAAAAGKERV0AAAAAAAAAiJEqZd0BoJCuuuoqp12tWjUT77XXXk6uS5cukZ8zaNAgp/3hhx+a+Mknn9yYLgIAAAAAAAAZcacuAAAAAAAAAMQIi7oAAAAAAAAAECOUX0DRe/bZZ02cqaSCb+3atZG5Cy64wGkfdthhJh47dqyTmzlzZtbHRMXRtGlTE0+dOtXJ9ejRw8T33XdfwfqEwthiiy2c9h133GFif275/PPPnfaJJ55o4hkzZuShdwAAAEC81K5d22k3bNgwq/386+nLL7/cxF9//bWTmzZtmoknTpxY0i4CecGdugAAAAAAAAAQIyzqAgAAAAAAAECMsKgLAAAAAAAAADFCTV0UHbuGrpR9HV2/rukbb7xh4p133tnJdezY0Wk3btzYxN27d3dyt9xyS1bHR8XSvHlzE/v1m2fNmlXo7qCAGjRo4LTPO+88E/tjoWXLlk77mGOOMfHAgQPz0DuUpRYtWjjtESNGmHjHHXfM+/EPP/xwpz1lyhQT//TTT3k/PsoX+1pn1KhRTu7SSy818eDBg53cmjVr8tsx5Kx+/fomfu6555zc+PHjTTxkyBAn9+OPP+a1X74tt9zSaR9yyCEmHj16tJNbtWpVQfoEoOwdffTRTvvYY481cdu2bZ1ckyZNsvpMu06uJDVq1MjEm222WeR+lStXzurzgXzjTl0AAAAAAAAAiBEWdQEAAAAAAAAgRii/gKLQqlUrEx9//PGR202ePNlp249szJ0718ktWbLExJtuuqmT++ijj5z23nvvbeK6detm0WNUdPvss4+Jly5d6uRGjhxZ6O4gz+rVq2fixx9/vAx7gvLsiCOOcNqZHvvLB7+00Nlnn23irl27FrQvKDz/+uWBBx6I3Pb+++838SOPPOLkli9fXrodQ85q167ttO3rYL/EwW+//WbiQpdbkNz+fP75507OPof6ZYm+//77/HYMGdWsWdPEfsm5PfbYw8SHHXaYk6NsBtazyxhK0iWXXGJiu0SZJFWrVs1pB0Gw0cdv2rTpRn8GUJa4UxcAAAAAAAAAYoRFXQAAAAAAAACIERZ1AQAAAAAAACBGyrymbpcuXZy2XTfll19+cXIrVqww8VNPPeXkfv31VxNTW6niadCggYn92jp2/TC/XuHs2bOz+vwrr7zSae+2226R27766qtZfSYqFruumCRdeumlJn7yyScL3R3k2T/+8Q+n3alTJxPvt99+OX/uIYccYuJKldzfy06cONHE7733Xs7HQGFVqfK/S7EOHTqUYU/+WsfyiiuuMPEWW2zh5Pxa4Ig/e36RpO233z5y22HDhpnYvj5H2dtqq61M/Oyzzzq5OnXqmNivmXzZZZflt2Mb0KdPHxPvtNNOTu6CCy4wMd/zylb37t2d9k033WTiHXbYIXI/u/auJM2bN690O4bY8s81PXr0yPsxp06damL/nTuInyZNmpjYPgdKf33fUtu2bU28du1aJzd48GATf/DBB06uPJ97uFMXAAAAAAAAAGKERV0AAAAAAAAAiJEyL79w++23O+0dd9wxq/3sx3Ak6Y8//jBxWdxCP2vWLBP7f6bPPvus0N2pcF5++WUT27ffS+7YmD9/fk6f37VrV6e9ySab5PQ5qLh23XVXp20/yuw/Hon4u/vuu522/3hPrk444YS0sSTNmDHDxCeffLKT8x+rR/nRrl07Ex944IFOzr+eyLfatWs7bbvU0Oabb+7kKL8Qf5tttpnT7t27d9b72mWDwjAstT5h47Vo0cLE9mOmvn79+hWgN9F23313p22XOhs5cqST4zqpbNmPx99zzz1Orm7duibONBfcd999TtsuQybl/h0N5Yf/2LtdRsF/lH306NEm/vPPP53cokWLTOxfa/iloN58800Tf/31107u448/NvEXX3zh5JYvXx55DJRPdilDf/6wvxP547Ak9t9/fxOvXr3ayX377bcmHjdunJOzx/rKlStzPn6uuFMXAAAAAAAAAGKERV0AAAAAAAAAiBEWdQEAAAAAAAAgRsq8pu55553ntPfaay8TT5kyxck1a9bMxHa9KMmtGXXAAQc4uZ9++snEO+ywQ9Z98+tozJkzx8QNGjSI3G/mzJlOm5q6hWXXldwYPXv2NHHTpk0zbmvX7LFjYL2rr77aadvjlDmiOLz22msmrlSpdH5nOm/ePKe9ZMkSEzdq1MjJ7bTTTib+5JNPnFzlypVLpT/YeHZNMEkaNmyYiadPn+7kbr755oL0ab3jjjuuoMdD2dpzzz2ddsuWLSO39a+JX3/99bz0CSVXv359p925c+fIbc855xwT299rCsWuo/vWW29FbufX1LXfj4HCu+qqq0xcp06dnD7Dr/V/5JFHOu2bbrrJxH793bKoUYns2DVu7fq2krT33nub+Pjjj4/8jI8++shp2+s8P/74o5Nr2LCh07bfa1Ra769A2bHXAi+55BInZ88hNWvWjPyMn3/+2Wm///77Tvu///2vif3v5/Y7SPbbbz8nZ899HTp0cHITJ0408eDBgyP7li/cqQsAAAAAAAAAMcKiLgAAAAAAAADESJmXX3j77bcztm2jR4+OzNWuXdvE++yzj5Ozb6Ped999s+7bihUrnPa0adNM7JeGsG/H9h+fRDwcc8wxTrtfv34m3nTTTZ3c77//7rSvvfZaEy9btiwPvUPc7Ljjjk67VatWTtueT5YuXVqILqGUtWnTxmnvsssuJvYfAcv2kTD/kR3/UbZFixaZuH379k6ud+/ekZ970UUXmXjQoEFZ9QX50adPH6dtP7roP45ql9vIF/v6xR/TPMpY3DI9pu/z5yKUH3fddZfTPvXUU01sfweSpOeff74gfYrSunVrE2+99dZO7rHHHjPx0KFDC9UlpOGXdzrrrLMit500aZKJf/vtNyd32GGHRe635ZZbOm27xMNTTz3l5H799dfozqKg/O/ETz/9tIntcguSW0IqU7kVn19yweaXuUS8Pfjgg07bLtOx1VZbRe7nrxl+9dVXJr7uuuucnL+mZzvooIOctv196ZFHHnFy9hqjP9cNHDjQxC+88IKTK0SpI+7UBQAAAAAAAIAYYVEXAAAAAAAAAGKERV0AAAAAAAAAiJEyr6lbWhYsWGDid999N3K7TDV7N8SuPWbX8JXcOh7PPvtszsdA2fFrnvo1g2z+3/HYsWPz0ifEl1+b0leI+joofXat5GeeecbJZar9ZJsxY4bTtmsvJZNJJ5epRrf/Oeeff76J69Wr5+Ruv/12E1etWtXJ3X///SZetWpV5PGQuy5dupi4Q4cOTu7777838WeffVawPq1n12L2a+iOGTPGxAsXLixUl1AghxxySMb8ypUrTZypZjfKVhiGTtv+d/zLL784OfvvNF+qVatmYr+24cUXX2xiv99nn312fjuGrPnvp6lRo4aJ33//fSdnX+/61xennHKKif2x0LhxY6e9zTbbmPill15yckcddZSJ58+fn7HvKH3Vq1c3sf0eGcl9J83cuXOd3J133mli3jlTcfnzwtVXX23ic88918kFQWBi/7uy/U6QO+64w8nl+n6aunXrOu3KlSubuG/fvk7Ofr+XX3e8rHGnLgAAAAAAAADECIu6AAAAAAAAABAjRVN+IR/q16/vtB944AETV6rkrof369fPxDwWEh8vvviiiQ8//PDI7Z544gmn3adPn7z1CcVhzz33zJi3H4dHfFSp8r/TZrblFiS3REvXrl2dnP+4Wrb88gu33HKLiQcMGODkNt98cxP7Y2/UqFEmnj59ek59QWYnnniiie2/C8m9tigEu4SIJHXv3t3Ea9ascXI33nijiSnNURwOOuigtHE69uOMX375Zd76hPw5+uijnfabb75pYr+kiv1oa0n45abatm1r4gMOOCByv+HDh+d0POTfZptt5rTtUhl333135H4rVqxw2o8++qiJ7fOgJO28886Rn+M/ql+IsiGI1qlTJxP36tXLyc2cOdPErVu3dnKLFi3Kb8cQC/Y5QZJ69uxpYrvcgiT9/PPPJrZLn0rSJ598ktPx7ZIKkrTDDjuY2F/jee2110zsl1u1+f1+8sknTVwW5cq4UxcAAAAAAAAAYoRFXQAAAAAAAACIERZ1AQAAAAAAACBGqKmbwSWXXOK069WrZ+IFCxY4uW+//bYgfcLGadCggdO268n59aPsOpd2XUFJWrJkSR56h7iza8edddZZTu6LL75w2v/5z38K0ieUjc8++8xpn3322SbOtYbuhti1ce06qZK077775uWYSG/LLbd02pnqSuZaxzJX559/vtO2a0NPmTLFyb377rsF6RMKpyRzQaHHJnJz7733Ou127dqZeNttt3VyhxxyiIn9moDHHntsTsf3P8euv+r74YcfTHzdddfldDzk3ymnnBKZ8+s02+8nyaRVq1ZZH/+jjz5y2nzvKluZ6q/b329mzZpViO4gZvyatv77G2yrV6828f777+/kunTpYuJdd9018jOWL1/utJs1axbZ9r+Tbb311pGfa/vtt9+cdlm/g4I7dQEAAAAAAAAgRljUBQAAAAAAAIAYofyC5//+7/9M3KtXr8jtOnXq5LS//vrrvPUJpeeFF15w2nXr1o3cdujQoSaePn163vqE4nHYYYeZuE6dOk5u9OjRTnvFihUF6RPyp1Kl6N+L+o8MFYL9CKzft0x97du3r4lPO+20Uu9XReSX89luu+1MPGzYsEJ3x9G4cePIHNcyxS/TI9ALFy502pRfiIfPP//cae+1114m3meffZzckUceaeKePXs6uTlz5pj48ccfz/r4Tz75pNOeOHFi5Lbjx483MdfW5Zd/nrJLc/glXOzHoPfcc08nd/zxx5u4du3aTs6fb+z8eeed5+TsMfbNN99k7DtKn/3Yu8+eUxKJhJN76aWXTPzll1+WfscQC++8847Ttkt72d+dJalhw4Ym/te//uXkMpX2sUs6+OUeMslUbmHt2rVOe+TIkSb+xz/+4eRmz56d9THzgTt1AQAAAAAAACBGWNQFAAAAAAAAgBhhURcAAAAAAAAAYoSaup4OHTqYeJNNNnFyb7/9tok//PDDgvUJG8euA9WiRYvI7caMGeO0/bpAwIbsvffeJvbr/gwfPrzQ3UEeXHjhhSb2ay2VtY4dO5q4efPmTs7uq99vu6YuSscff/zhtO1acna9S8mtvz1//vy89Kd+/fomzlQbb9y4cXk5PsrOwQcf7LS7desWue2iRYuc9qxZs/LSJ+TXggULTGzXLvTb11xzTakcb+edd3badn13v47mVVddVSrHRH699dZbTtueG/y6uXaN20w1L/3PvOSSS5z2K6+8YuK//e1vTs6uX2lfh6Ew6tWrZ2L/GtJ+h8ANN9zg5Pr06WPiwYMHO7mPPvrIxHYdVUn6/vvvTTx58uSMfdt9991N7K/PcA4rH5YvX+607VrbtWrVcnL2O63sd11J0rx580w8c+ZMJ2ePQ/v7uCTtt99+JexxypAhQ5z2ddddZ2K/JnhZ405dAAAAAAAAAIgRFnUBAAAAAAAAIEZY1AUAAAAAAACAGKnwNXWrVavmtI888kgTr1y50snZNVZXrVqV344hZ3Xr1nXadv0Tv06yza/7tWTJktLtGIrONtts47Rbt25t4m+//dbJjRw5siB9Qn7ZdWvLgl3XbLfddnNy9lyXyZw5c5w257PS59cPmz59uok7d+7s5F599VUTDxgwIKfj7bHHHk7br3G54447mjhTzcPyVicaG8+/JqpUKfp+jv/85z/57g6KkF9H055j/Lq9/vkH5ZNf3/2kk04ysf+OiC233DLyc+677z4T+2NhxYoVTnvEiBEmtutqStIRRxxh4saNGzs5+/yK/LjzzjtNfMUVV2S9n32+ufjii52c3y4N/vxivy+na9eupX48bDy/Nq3/bz8XTzzxhNPOVFPXfweGPb4fe+wxJ7dmzZqN7lu+cKcuAAAAAAAAAMQIi7oAAAAAAAAAECMVvvxCz549nXbz5s1NPHr0aCc3fvz4gvQJG+fKK6902vvuu2/kti+++KKJ7fIaQDbOPPNMp12/fn0Tv/766wXuDSqC3r17m/iSSy7Jer8ff/zRxGeccYaTmzlz5kb3C5nZ55cgCJzc0UcfbeJhw4bl9Plz58512n6Jha222iqrz/EfNUP8denSJTLnP/b44IMP5rs7KAInnnii0z799NOdtv0467x58wrSJ+TXW2+9ZWJ/TunWrZuJ/TnFLs3hl1vw9e/f38TNmjVzcscee2zaz5T+ek2D0mc/Ev/ss886uaefftrEVaq4S0s77LCDiTOV/iktdokyyR2rffr0cXI33nhj3vuDwrn66qtNXJJSGxdeeKHTzvU6vKxxpy4AAAAAAAAAxAiLugAAAAAAAAAQIyzqAgAAAAAAAECMVLiaunbtOkm6/vrrnfbixYtN3K9fv4L0CaXriiuuyHrbSy+91MRLlizJR3dQxBo1ahSZW7BgQQF7gmL12muvOe1ddtklp8/55ptvTDxu3LiN6hNKburUqSY+6aSTnNw+++xj4iZNmuT0+cOHD8+Yf/zxx03cvXv3yO2WL1+e0/FRvmy//fYmtutd+mbNmuW0P/vss7z1CcXjqKOOyph/5ZVXTDxhwoR8dwcFZtfXTdfOlX3+8eu22jV127Vr5+Tq1Klj4vnz55dKX+Bas2aNif3zRNOmTSP3O/TQQ028ySabOLm+ffuaONP7bzaG/Q6Dli1b5uUYKBvnnnuu07ZrJvu1nX2TJ0828YgRI0q3Y2WEO3UBAAAAAAAAIEZY1AUAAAAAAACAGKkQ5Rfq1q1r4n/9619OrnLlyk7bftT1o48+ym/HUObsR3ZWrVqV8+csWrQo8nPsx0223HLLyM+oVauW0862jIT9SIwkXXPNNSZetmxZVp+B3BxzzDGRuZdffrmAPUGh2I9yVaoU/XvRTI+nDhkyxGlvu+22kdv6x1i7du2GuphWx44dc9oP+ffll1+mjUvTDz/8kNV2e+yxh9P++uuv89Ed5NlBBx1k4kzz1IsvvliI7qDI+Oe3pUuXOu277rqrkN1BEXruueectl1+4eSTT3Zydik9SieWL2+//XZkzi495ZdfWL16tYkfffRRJ/fQQw857X/+858mzlRuCPG33377mdg/z1SvXj1yP7/E5oUXXmjiP//8s5R6V7a4UxcAAAAAAABEcNG5AAAgAElEQVQAYoRFXQAAAAAAAACIERZ1AQAAAAAAACBGirKmrl8nd/To0SbeaaednNz06dOd9vXXX5+/jqHcmTRpUql8zvPPP2/i2bNnO7mtt97axH4dqHz49ddfTXzTTTfl/XgVzcEHH2zibbbZpgx7grIwaNAgE99+++2R273yyitOO1Mt3JLUyc1228GDB2f9mSh+di1oO/ZRQ7c42O+S8M2dO9fE9957byG6gyJg1yC0r2sl6ffff3faEyZMKEifULz8ax37euu4445zcolEwsTPPPOMk5s2bVoeeofS8Oabb5rY/75apcr/lqjOO+88J9ekSROn3bZt26yON2vWrBL2EOWN/X6QGjVqRG7n13m3a3JL0gcffFC6HSsHuFMXAAAAAAAAAGKERV0AAAAAAAAAiJGiLL/QuHFjp92yZcvIba+44gqn7ZdjQPy89tprTtt/TCcfTjzxxJz2W716tYkzPVY9atQop/3ZZ59Fbvv+++/n1Bdk5/jjjzexX+rliy++MPF7771XsD6hcEaMGGHinj17Orl69erl/fhz5swx8ZQpU5zc+eefb2K/DAwqtjAM08YoTkcccURkbubMmSZetGhRIbqDImCXX/DnkFdffTVyP/8R2dq1a5vYHotAJl9++aWJb7jhBid3xx13mPjmm292cqeddpqJly9fnqfeIRf2Nexzzz3n5E466aTI/dq1axeZW7NmjdO256ZevXqVtIsoY/754+qrr85qv6eeesppjxkzprS6VG5xpy4AAAAAAAAAxAiLugAAAAAAAAAQIyzqAgAAAAAAAECMFE1N3UaNGpn4zTffjNzOr4H4yiuv5K1PKBsnnHCC07brr2yyySZZf87uu+9u4pNPPjnr/R555BGn/eOPP0Zu+8ILL5h46tSpWR8DhbP55ps77Q4dOkRuO3z4cBP7dZ1QHGbMmGHirl27OrlOnTqZuEePHnk5/k033WTigQMH5uUYKD5Vq1aNzFFnMP78axv/3RK2FStWmHjVqlV56xMqDv96p3v37ia+/PLLndzkyZNNfMYZZ+S3YyhKTzzxhNO+4IILTOx/B+zXr5+JJ02alN+OoUTsa49//vOfTq569eombtWqlZOrX7++07a/Zz/55JNOrm/fvhvZSxSa/Xf/zTffOLlM6zj2v29/PFUE3KkLAAAAAAAAADHCoi4AAAAAAAAAxEjRlF84//zzTdywYcPI7caOHeu0wzDMW59QPtx+++0b/RndunUrhZ4gjvzHUxcsWGDiUaNGObl77723IH1C+fDee+9Ftv0yQPY5qmPHjk7OHkdDhgxxckEQOG3/USQgG2eddZaJFy5c6OT69+9f6O6glK1du9Zpf/bZZybeY489nNz3339fkD6h4jj33HOd9jnnnGPihx9+2Mkx32BjzZkzx2kfdthhJvZL3l1zzTUmtsuCoHz57bffnLZ9nXzaaac5uQMOOMBpJ5NJE//+++956B0KqX379ibefvvtnVymdTu71I9dZqqi4E5dAAAAAAAAAIgRFnUBAAAAAAAAIEZY1AUAAAAAAACAGIltTd2DDz7YaV922WVl1BMAxcyvqXvQQQeVUU8QJ6NHj87YBgrp008/NfGAAQOc3Lvvvlvo7qCUrVmzxmn37t3bxH4Nus8//7wgfUJxufTSS03cr18/J+fXlx80aJCJ7fcQSNLKlSvz0DtUZDNnzjTxW2+95eSOPfZYE++2225OjncUxMOTTz6ZsY3iYtddz1RD94477nDaFf1aljt1AQAAAAAAACBGWNQFAAAAAAAAgBiJbfmF1q1bO+3q1atHbjt9+nQTL1myJG99AgAAKG86duxY1l1AAf3yyy8mPvvss8uwJygW48aNM3H79u3LsCdAtC5dujjtiRMnmrhJkyZOjvILQPlTp04dEwdB4OR+//13E99zzz0F61MccKcuAAAAAAAAAMQIi7oAAAAAAAAAECMs6gIAAAAAAABAjMS2pm4mdv0cSTr00ENNPH/+/EJ3BwAAAAAA5MnixYud9k477VRGPQGQiwEDBqSNJal///4mnj17dsH6FAfcqQsAAAAAAAAAMcKiLgAAAAAAAADESBCGYfYbB0H2GyPfPg/DsFVZdyIbjJvyIwzDoKz7kA3GTLnCXINcMG6QC8YNcsG4QS4YN8gF4wYlxndw5CDruYY7dQEAAAAAAAAgRljUBQAAAAAAAIAYYVEXAAAAAAAAAGKkSgm3nytpRj46ghJrVNYdKAHGTfnAmEEuGDfIBeMGuWDcIBeMG+SCcYNcMG5QUowZ5CLrcVOiF6UBAAAAAAAAAMoW5RcAAAAAAAAAIEZY1AUAAAAAAACAGKlQi7pBMtgtSAafZbntsUEyeCbffUL5FySDekEy+DZIBlWz2HbrIBlMCZLBZoXoG8ovxg1ywXkKJcVcg1ww1yAXzDfIBeMGuWDcIBcV8fqmpC9KK1eCZFBH0sOSDleqqPO1YSJ8OsMu/SXdmc3+YSIcFSSDm4NksFeYCCfl68+AshEkg66SEpIaSvpV0plhInw/YvNekh4NE+EKa//DJN0uaRdJ8yVdGSbC58JE+FuQDN6VdL6k+/L5Z0BhBcmgmaSBklpKmiOpZ5gIR2bYxRk3QTLYTtIDklpLWibpxjARDpYkxk3x4jyFkgqSwaWSzpS0p6RhYSI8cwO7+HPNSZL+KWkfSZ+EibDt+g2Za4pXkAzGSDpA0up1P/o5TIS7ZNjFn2si92euKV5c2yAXQTJY4v2omqQHwkR4WcQu/riZLPclQFUlvR4mwo6Mm+IVJIOhkg6VtIVS379vDxPhvzPs4o+b2yWdImlLSQskDQkT4U0S801FECSDv0n6StLwMBGemmFT5/om0/7Fcn0T9zt1B0paKWlrSd0lDQqSwe7pNgySQQNJ7SS9WIL9hyk1MaCIBMng75Juk3SWpBqSDpH0Q8S2m0k6Q9JQ62e7SXpaUm+lTir7SPrc2u0pSRfko+8oG0EyqCLpJUmvSKqj1LwwNEgGTSO2/8u4WRf/V6n55mhJNwfJoJ2VZ9wUJ85TKKlfJN0o6ZENbRgx18yXdI+kWyN2Y64pXpeGibD6uv8iF3Qj5poN7c9cU2S4tkGurHmiulJ/98slPZ9u23TjJkyEu1v715A009ufcVOcbpG0Y5gIa0o6VtKNQTJomW7DiPnmYUm7rtv/IEndgmRwgpVn3BS3gZI+zbRBhuubTPvH/vomtou6QTLYQlJnSdeHiXBJmAjHSRol6bSIXf4uaYL1m55s9h+j1AUKiktSUr8wEX4UJsK1YSL8OUyEP0dsu7+khWEinGX9rI+kB8NE+HqYCFeHiXBemAinW/mPJe0cJINGQrHYVdK2ku4OE+GaMBG+I+kDRc83zrgJkkF1SW0l3RQmwlVhIpwoabiks619GDdFhvMUchEmwhFhInxR0rwsNv/LOSpMhG+FifA5pRaH02GugTPXZGmMmGuKDdc2KA1dJP0uKeqJx3TfpWyHSKov6QXrZ4ybIhQmwslhIvxzfXPdf40jNk93ffNtmAiXWtusldTEajNuitS6p6wXSnp7A5umvb7ZwP5jFPPrm9gu6kpqKmlNmAinWT+bKCntHVBKPcb4bQn3nyJpxyAZ1CyF/qIcCJJBZUmtJNULksH3QTKYFSSD+4NkUC1iF3/cSKnHExUkg6+CZDA7SAZD1z0iLUkKE+FqSd9L2jsPfwSUjSDiZ3tEbO+Pm8D737/sz7gpSpynkG/pzlEZMdcUtVuCZDA3SAYfBMmgbYbtosZNpv2Za4oP1zYoDWdIeiJMhGFEfkPnqTOUehzaLNYxbopXkAweCJLBMklTJc2W9FrEpmnHTZAMeq0r/zFLqTIOpqQZ46Y4rbvu6Cfpyiw2/8u4yWL/2F/fxHlRt7qkRd7PFin1CEc6tST9UcL9129fK8c+ovzZWtImSv1WubVSpROaK3X3bTr+uJGk7ZW6i6GzpL8pVUfKr93zhxg3xWSqUnch9AySwSZBMjhcUhtJm0ds74ybMBH+odTdL9cHyaBqkAxaKDV+/P0ZN8WF8xTyLd05KhvMNcXnGkk7S9pO0hBJLwfJIOoOqHTjZkP7M9cUH65tsFGCZNBQqTHzeIbNIs9TQTLYXKnvZI+lSTNuilCYCC9W6jq2taQRkv6M2DTtuAkT4a3r9m8h6Un99TqZcVN8+kt6OEyEP2Wxbbpxs6H9Y399E+dF3SWS/NX0mor+crNA7hfhbPZfv/3CHPuI8mf5uv+9L0yEs8NEOFfSAEkdIrb3x836z3g0TITTwkS4RNLNafavIcZN0QgT4SpJnZR6NONXpX7T95xSvyVOJ9246S5pJ0k/SRqkVN0nf3/GTXHhPIV8SzfXZIO5psiEifDjMBH+ESbCP8NE+LhSi21ZX9tksT9zTZHh2gal4HRJ48JE+N8M22Q6T52gVB34sWlyjJsita7cyzilbpS6KGKzyHETJsIwTIRfKPWdPOmlGTdFJEgG+0g6TNLdWe7ijJss94/99U2cF3WnSaqy7i126+0taXLE9pOUepS1JPs3k/RjmAgXl0J/UQ6EiXCBUhebUY8I+fxxs/5nkfuve/FEE6Uek0aRCBPhpDARtgkTYd0wER6h1B1Nn0Rs/pdxEybCGWEiPCZMhPXCRLi/pLr2/oybosR5CvmW7hyVEXNNhREq/eP1Unbjxt+fuaYIcW2DjXS6Mt+lK2Web9KWbmDcVBhVFF1TN5vzlLM/46YotZW0o6SZQTL4VdJVkjoHyWBCxPb+uMlm/9hf31Qp6w7kKkyES4NkMEJSvyAZnKvUY/THKfUmxHT+I+neIBlUDRPhiiz3byPp9fz9KVBGHpV0WZAMRktaJemfSr35N51PJNUKksF21svUHlXqUbOhSt3ZcI23/35KTQwz8tJ7lIkgGeyl1CJbJUkXS2qg9I+LSWnGTZAMmin1C4U/JZ0k6XClTiLrMW6KDOcp5GLdl5IqkipLqhwkg6qSVq+rFedLN9dUVqrMUBVJldbtv2bdXXkSc03RCZJBLaVeKjNW0mpJJyv18qF/RuzizDVZ7s9cU4S4tkGugmRwkFLlWp7fwKbpvkspSAbbK/WW+gvT7MO4KTJBMqgvqb1S35mXK3X35CmSukXs4oybIBlUknSeUk8TLJS0r6RLJN1i7cO4KT5DJD1jta9SapE26g5v5/omy/1jf30T5zt1pdTFRzWl6kENk3RRmAjT3gEVJsLfJL2j1BfibPc/RdKDeeg3ylZ/SZ8qdRE7RdIXkm5Kt2GYCFcqdXF7qvWzRyQ9odQbNmcodSH7D2u37pIG56HfKFunKVXQ/3dJh0r6u/UGV0e6cSPpCEk/KPVYyIWSjgwT4Rwrz7gpTpynUFJ9lPrC00upOWS5Iuq+R8w1p63bZ5BSNeuWS3rIyjPXFJ9NJN0oaY6kuZIuk9QpTIRpX06UZq7JZn/mmuLEtQ1ydYakEetqK0eKGDdSaux9GCbC6Wl2Y9wUn1CphbRZSs0Xd0r6Z5gIX0q7cfpxc7yk6UqVIRuq1Dtt7PfaMG6KTJgIl4WJ8Nf1/ylVmm6Fd56xt3eub7LcP/bXN0EY+aLK4hMkg92UekRkvwxv6Fy/bUdJp4WJ8KSCdA7lVpAM6kl6X1LzMBEu38C29ZW606X5ut8OoYJi3CAXnKdQUsw1yAVzDXLBfINcMG6QC8YNclERr28q1KIuAAAAAAAAAMRd3MsvAAAAAAAAAECFwqIuAAAAAAAAAMQIi7oAAAAAAAAAECNVSrJxEAQU4C0/5oZhWK+sO5ENxk35EYZhUNZ9yAZjplxhrkEuGDfIBeMGuWDcIBeMG+SCcYMS4zs4cpD1XMOduvE1o6w7AKBCYK5BLhg3yAXjBrlg3CAXjBvkgnEDoBCynmtY1AUAAAAAAACAGGFRFwAAAAAAAABihEVdAAAAAAAAAIgRFnUBAAAAAAAAIEZY1AUAAAAAAACAGGFRFwAAAAAAAABihEVdAAAAAAAAAIgRFnUBAAAAAAAAIEZY1AUAAAAAAACAGKlS1h0ACmmzzTZz2h988IGJmzdv7uRefvllE3fq1Cm/HQMAAAAAAACyxJ26AAAAAAAAABAjLOoCAAAAAAAAQIwUTfmFgw8+2MQffvihk9tll11MfMwxxzi5o48+2mm/+uqrkccYP368iceNG5dTP1F4dsmFu+++28nts88+Jg7D0Ml9/vnn+e0YAAA56tu3r4kTiYSTGzNmjNNu165dAXqE8qhly5ZO2y4n1blzZydnXy9LUhAEJvavkSZMmGDiKVOmOLmbb77ZxFOnTi1hjwEA5VX16tWd9vbbb2/iiy++OHK/Rx55xGl/+eWXpdsxoALjTl0AAAAAAAAAiBEWdQEAAAAAAAAgRljUBQAAAAAAAIAYiVVN3Zo1a5r4qaeecnLt27c38fLly53cpptuamK/DoyvdevWkTn7c5ctW+bkLrroIhMPHz484zFQWP/4xz9MfP755zu5d955x8Q33HCDk/voo4/y2zEAFVLt2rWdtl3b+6ijjnJyPXv2dNpr1641sX+umTFjhonvuusuJ/fbb7/l1lmUW23atInMtW3bNrLt19tFPPjXL7vuuquJM127tmjRwmnbtXHtmrl+TpKGDBli4pEjRzq5N998cwM9BgAUA3v9xL8u7dOnT1afceGFFzrtZ5991sQ9evRwcvPnzy9pF4FS8cwzz5j45ZdfdnL++mN5wp26AAAAAAAAABAjLOoCAAAAAAAAQIzEqvzCbbfdZuKjjz46crtq1ao57SlTpph4zpw5Tm7x4sWRn+M/lmYf0z/Gww8/bOJp06Y5uUmTJkUeA/m3zTbbRObeeustE1NuAUBp2WSTTZz2lVdeaeJLLrnEyTVo0CDyc+xyC5L7eHTnzp0j99tqq62c9tlnnx3dWcSSX2Ih220pvxBPgwcPdtr2XOCXBJs6daqJ77333sicf03sl1hAxWLPEyeccIKTs8832267rZObMGGC037++edNfOutt5ZiDwGUhWuvvdbEvXr1yukzKleu7LS7detmYruMpiSdddZZJqbUD/KpUiX3Hld7LH7zzTeF7k7OuFMXAAAAAAAAAGKERV0AAAAAAAAAiBEWdQEAAAAAAAAgRsp1Td3dd9/daXfp0iVy21mzZpn49NNPd3Lff/+9iRcuXOjklixZEvmZfo2NG264wcR9+vRxcjVr1jRxIpFwcueee66JFyxYEHk85EeNGjVMvGrVKidn19QFSmqfffZx2v379zdxhw4dnJw/n9i1UocPH+7kevfubeLZs2c7uXbt2pn47bffdnLLly/PptsogAsuuMBp33jjjTl9ztixY532IYccktV+/nmQmroVW9++fcu6C9hII0aMcNqdOnUysV0nV5L23XffgvQJ8WO/Z8IfU/vtt5+J/feK2N+zvv32WyfXsGFDp22f72bMmOHkhg0bVsIeY2McddRRTvvFF180sV/7PxP7+nLUqFGR2/l/33ZN7/3339/JzZ0712mPGzcu6/6gsH788cfInF3ffeDAgU5u8uTJJvbHW79+/Uzsv//mpZdeMrH9TiVJuv322522X1MeKInmzZs7bf+dJHHBnboAAAAAAAAAECMs6gIAAAAAAABAjJTr8gv2Y/OSVLduXRPbt/pL7q35Y8aMKZXj249HS+7ji5tuuqmTu+qqq0x8/PHHO7lHHnnExK+++mqp9A3Rtt12W6d9zjnnmHj8+PFObsKECQXpE+LLf1yoTZs2Jn700UedXIMGDUzsz1H+fGLnO3fu7OTsx9x22GEHJ9e2bVsTn3HGGU5u6NChf+k/CscuGXT99dfn9Bm9evVy2vaji5L7uFrPnj1zOgaA+LnoooucdsuWLU3cqFEjJ2c/Dj9z5sz8dgzlmv8oqf09xC8hZY8Vv4TQxx9/bOJFixY5Of86xX50+sQTT3Ryzz77bGTuiy++MPF3333n5PxrKmTHnxtKUnLBVq1aNROffPLJWe93+eWXRx7bvy62x5hfluybb74xsV8KwC8HgtJnl/vxPf/88ybu0aNH1p85ceJEE48cOdLJ1alTx8T+9XTjxo2dtl1ezC+ziPhp2rSpie+8804nd9lll5nYL/WSD1999VXej1FauFMXAAAAAAAAAGKERV0AAAAAAAAAiBEWdQEAAAAAAAAgRsp1Td3NNtssMvf444877YEDB+a7O47rrrvOadv1hXbaaScnd8IJJ5iYmrr516dPn7LuguOAAw4wsV93zGbXFpKkadOm5a1PyF6LFi2c9ujRoyO3nT17tokvvfRSJ7ds2bLI/fyaZ0uXLjXxfffd5+RWrlyZ9ngoPLuGriTdcsstJvbrGNr1AP06UMcee6yJp0yZ4uT8mnM33HCDif0aZKNGjYo8/qRJk0y81157CfGXTCZNnEgkMm5rvxPAjhEfc+bMcdpDhgwx8Y033ujk7H//1NSt2Pza63Yd3V9++cXJ7bLLLia2rzU25KeffnLadq3cP//808l16NDBxE8//XTkZ1avXt1p2+8aQPYefvhhp23XHG3SpImTyzRXVK1a1cTHHXdc1sdv1qyZievVq+fkKlVy7y078MAD08a+FStWOO077rjDxBs6FyI39r9b/7rUP/9ka9y4cSb2x5R9PX3wwQc7uW7dukV+5llnneW0V69enVPfUHbsdZNjjjnGydnrf6VVU9efB20///xzqRyjELhTFwAAAAAAAABihEVdAAAAAAAAAIiRcl1+oX///pG5jz/+uIA92bA33njDxBdeeKGTs28jR/4dffTRkTn/MaTSMmjQoMjj165d28TVqlWL/IzFixc77bvvvtvEmf4toPTZj9Xbj7T73n77bad97bXXmnjChAlZH2/bbbd12i+99JKJa9Wq5eTsx8z846Ow/NIc9r99/7FC+1HWBx54wMlNnjw562Paj05+8sknTu6xxx4z8ZVXXunk9txzTxPbj21L0vnnn5/18VF+8JhpxWbPMUEQODn7kWc/l4lf/iVT2SCUX127djXxFVdc4eTmz59vYnucSCUruZDJ9OnTTbzbbrs5uSeeeCJyP/vax3/EHrmxrxmk0vkeZH8/2ZA99tjDxH//+98zbms/Vt+yZcvI7exSEJLUo0cPEw8YMMDJLVq0KKt+IrO33nrLxO3bt3dydsm4XI0fP95pX3311Sb2S1fa36sld9y8/PLLTu65557b6L6hsPzxZctHOQT/O9DChQtNXJLv8mWNO3UBAAAAAAAAIEZY1AUAAAAAAACAGGFRFwAAAAAAAABipNzV1N15551N7NeZtOvifPXVVwXrUzbeeecdE/s1dZF/m2++uYmrVHGHtV1/xa45uSH25/i1M0eOHOm0t9lmGxP7tTTnzJljYrsmkf+5DRs2dHJ2jRe/BtmMGTMy9h0b5/rrrzfxVltt5eTs2k5+rbrvv/8+p+PZNcckqXnz5pHbjh49OqdjoPQdddRRTjsMQxOvXbvWyY0ZM8bEd911V17606tXr8i+2WOsVatWeTk+gPypV6+e0z733HNNbM89kvT444+b2K+pa2/r5/xrm6eeeioyh/Jrr732MrF/TWrXcF+yZEne+zJr1qyst/3jjz9M7I9pxNPXX3+dNk7Hfj/Jdttt5+Ts65tzzjnHydWsWdPE/vsEbrjhhuw7i0h2vfVMNU999nnKrn0rSQ8++GBWnzFs2DCnffHFF0du+7e//S3rvqF8qFGjhtM+9NBDTezXRPbfJVIaNtlkE6dtf39bvXp1qR8vX7hTFwAAAAAAAABihEVdAAAAAAAAAIiRcld+4dRTTzWxXYpBkl544QUTjx8/vmB9QvlnP96x9dZbO7khQ4Zk9Rl+uQ+7/EGfPn0y7vvLL7+Y+Mknn3RyDzzwgIkzPYY2atQop92hQwcTN2jQwMlRfqF0PfTQQ077xBNPNPHSpUudnP0IWK7lFiT3cY9rr73WydmPxI4dO9bJ+W0UVt26dU283377Zb2fPy/km3+82267raDHB7Dx7JIL/txvl2yaMGGCk7MflR03blzk55933nlOu2XLlk77hBNOMLH/OLw9/9nHk6Rly5ZFHhP517hx48hcoc8FRxxxhNOuVq1a5Lb+o7aoWFasWGHi6dOnOzl73PrlF+yyHSUps4fsffbZZ5E5u9xL1apVndz9999vYv8x9zZt2pRS7/7HXg+QpG+//dbE//nPf5ycXdYTZWe33XZz2nbplY8//tjJ+aXtclWrVi0TN2vWzMn54yQuuFMXAAAAAAAAAGKERV0AAAAAAAAAiBEWdQEAAAAAAAAgRspdTd2uXbua2K91cu+99xa6O4iJ5s2bR+a+++67rD7Dr5t7wQUXmNivJffOO+847csvv9zEkydPzup4vmz7idLXqlUrp23/fS9ZssTJffPNNzkdw68l1b9/fxO3bt068vj9+vXL6XjID7vm5I477hi53fvvv++0X3311Xx1qcRq167ttO2a3bNnzy50dwBE2GWXXdLGkjRixAgT23XgS8J/58BWW23ltO33XHTq1MnJffLJJyb2z4t2f6ZOnZpT35C9zTff3Gkff/zxkdva74DIl0033dTEN998c2TOv776+uuv89sxxNZxxx0XmatRo4aJu3Tp4uRuv/32vPWpInnxxRdN7Nc1tb8T+++1sesk+9+D8sGuNS9Jzz77rIn9Wu/2u3NeeuklJ0dd+MI5+OCDI3P5eo/MySefbGL7XSmS9N577+XlmPnGnboAAAAAAAAAECMs6gIAAAAAAABAjJS78gs2/5GtcePGlVFPUN5tu+22Oe3XtGlTE9u34vseeughp92jRw+nvXLlypyOn8mECRPSxogH/9H8iy++2GlfccUVkfvaj8B/+eWXpdovbBy7/EImiUTCaS9YsCAf3cnJDjvs4LT32GMPE1N+oTj17du3rLuAHNjXvZUrV8778ebOneu077nnnrSx5D66ei5QoHAAAA+KSURBVN555zk5+/HFo446ysl9/vnnG91PZFaIsWLzH6tu3769iXfeeefI/R555BGnPWPGjNLtGGLLHzeZzmGLFy82sf99DaXD/v946NChkdv5JVW6d+9u4pNOOsnJ1alTx8QdOnTY2C5ukF+mxv5z+KVfunXrZuJcyyoi2mabbWZi//vx/PnzTWyXh5Okf//73yb2S31sscUWJj7kkEMyHj8Igshc1apVM+5bXnGnLgAAAAAAAADECIu6AAAAAAAAABAjLOoCAAAAAAAAQIyUeU1du/6F9Ne6TEA2atSoYeJMdVJ8l112mYlr1arl5J5++mkTX3TRRRvRu+zYfwZJWrVqlYnzUbMX//PNN9847T333NPEdevWdXJffPFFVp+51VZbOW2/7nMYhpH7vv322yZeuHBhVsdDYdg1uTLNNWPHji1Ed7JWqdL/foe7du3aMuwJgGIwZMgQE48YMcLJ2fPfq6++6uTs66mRI0fmqXcVy+rVq532jz/+aGK/vv/hhx9u4okTJ+Z0PL/O4Wmnnea0b7nllqw+57HHHsvp+Ch+HTt2dNr+eoHNrqNbnt5fAHf+988Fdu1v/zuwza+d6n9/+v333yP3TSaTJj777LOdnH09b79bQpIGDBhg4muuucbJ8a6TjWfXrd1pp50it3v55Zedtv39ZcqUKU7OPu+9/vrrGY9/6KGHpu2LJN18880mnjdvnpN74oknMn5uWeJOXQAAAAAAAACIERZ1AQAAAAAAACBGWNQFAAAAAAAAgBgp85q6J510ktNu3LixiefOnVvo7uTs2GOPjcz5ta5Q+uz6OplqlfrsumD+fn7NsHyw66yec845Ts6vUYf8Offcc512zZo1TdyhQwcnZ9fbLQl/jjj99NNN3LlzZyc3ePDgnI6B/Nt3331NXJK5pqzZdaji1G8A5Z9/vW7Xzb3rrruc3IMPPmjiRo0aObl77rknD70rfv57F9q0aWNi/50Bt912m4nt+rqS9MILL5h4t912c3J2zcvWrVs7Ob/m5eLFi0285ZZbOrmZM2ea+KeffhKwXpMmTUx84403Rm63dOlSp/3www/nrU/YOPb7RZo2berkxo8fb+JM7w/ZmHeL9OjRw8TPPvuskxs0aJCJ/Zq6hx12mIn9GuFHHXVUzv1Byp9//mni7777zsnVr1/fxHZ9W0l6/PHHTZyplvKG2Oeh7bff3snZ7zS64IILnBw1dQEAAAAAAAAApYJFXQAAAAAAAACIkTIvvxBXLVu2dNrHHHNM5LbXXXddvruDHNm31f/f//2fk7Pb1157rZMbMmSI0543b15Ox7dLLCxbtszJ+Y8sIn+WL1/utDt27Gjitm3bOrlWrVpFfs7kyZNN/Prrrzu5gQMHOu0uXbqYeNq0aU5u+vTpmTsMbIQlS5Y47VznLwBI57333jOx/6jq2LFjTXznnXc6OcovlI5Zs2aZ+NRTT3VyvXv3NnH79u2dnN22H0GVpP/+978mHjNmjJMbNmyY037llVdM7Jf7efvtt008f/78tP1HxWA/mi+588EWW2wRud8NN9zgtKdOnVq6HUPO7O9Pkjun2yUHJalr164mfumll/LbMbnlHiTp4IMPNvGECROc3M4772ziAw880MkdeeSRJh49enRpdrHCWLFihYntsnaSVKXK/5YnS+scsd122znt2rVrm3jixIlO7owzzjCxvzZTnnGnLgAAAAAAAADECIu6AAAAAAAAABAjLOoCAAAAAAAAQIxQU7cE7Dq6V1xxhZOrVauWiT/44AMn98Ybb+S3YxWQX5enQYMGOX2OXUuyRYsWTm7UqFEm7t+/v5Oz6+lIbk3lP/74IzLXp08fJ9e8eXMT33jjjU7uo48+yth3FIZfO85vZ+vCCy902naduU8//dTJzZkzJ6djAOudfvrpkbm+ffs6bb+WGOLBnov82t8+++/c//sH8mnu3LlOe9y4cSbeddddC92dCse+lpXcev/++0FsK1eudNqZzhNNmzZ12ptuumnktsOHD4/MoWLp1auX0z722GMjt/3hhx9MfO+99+atT9g41atXd9r293V/XnjhhRdMbNe3lQrzHdj+vn7KKac4uQ8//NDENWrUcHLXXHONiampu/EWL16c92P46zZ2zW67BrwkTZo0Ke/9yQfu1AUAAAAAAACAGGFRFwAAAAAAAABipMzLL/z4449O2390vSxVrlzZaV911VUmPvnkk53czz//nHY7SVq9enUeelex/fLLL077u+++M3GjRo2cXPv27U384IMPOrlly5aZePbs2U5u3333NbFdQkGSpkyZ4rTt8ht33XWXkzvnnHPSHk9ySy74JR4QbzvuuGPG/JIlS0x8zz335Lk3KC3244L+Y1dbbbWViR955BEnd/bZZ+e3Yx67L5Jb0mPw4MEF7QuAissvsdCpUycTf/PNN4XuToW3atUqE5fWI87bbbdd1tt+/PHHpXJMxE/Xrl2d9uWXXx657dKlS522PW+sXbu2dDuGUjNs2DCnbc8Nt912m5MLgsDE/ppLoe29995O2+6bL66P51dktWvXjszlWlaxvOFOXQAAAAAAAACIERZ1AQAAAAAAACBGWNQFAAAAAAAAgBgp85q67777rtO2a9PWrFnTydk1AufOnVsqx99rr72c9sUXX2ziFi1aOLlWrVpFfs6pp55qYupFFZ5dt/bVV191ch06dDDxG2+84eQGDBhgYr+mrm3//fd32tdee21k3q/D8+2335q4d+/eTm7kyJGRx0S8XX/99RnzL7/8soknTJiQ7+6glHz55Zcm7tmzp5N77LHHTHziiSc6ufvvv9/E+fr7fuihh0y89dZbO7nnn3/exCtWrMjL8ZFfbdu2zdhGcfPrT9p1socOHVro7mRkv9vgpptucnKbb765if15EvHUpUuXsu4Cyqk2bdqY2H+vSaa6pWeeeabT/vrrr0u1XyiMIUOGmPjII490cu3atTPxE0884eTGjh1r4ltvvdXJTZs2Lae+9OjRw2mfe+65Jm7cuLGTyzQ2UVz+/PPPsu5CqeBOXQAAAAAAAACIERZ1AQAAAAAAACBGyrz8QibNmjVz2qNHjzZxpkflS+KAAw5w2nXr1o3c1i75MGrUKCf36aeflkp/kJtZs2aZ2H+8wy7xceCBBzo5+5Fkn/3oRRiGWffl0UcfddrXXHONiefNm5f15yB+dt99dxN37tw547Z+KRDEzwcffOC0n376aRN369bNydmPIJZW+QX70TVJOv744038+++/O7l+/fqVyjFRdhKJRFl3AQVm/5u+8847nZz9WGu+yi/Uq1cvbV98fs4uX+bPRaeffrqJp06durFdRBlo2LCh0z7llFMit33vvfec9uLFi/PSJ5QPtWrVctqvvPKKibfYYouM+w4cONDE/vdsxJP9771Tp05ObuLEiSZu0KCBkzvjjDNMfNpppzm5tWvX5tSXKlVyW/by13i4nkZ5xJ26AAAAAAAAABAjLOoCAAAAAAAAQIywqAsAAAAAAAAAMVLuaur27t3bxH369HFydo2ufLHrtMyfP9/JDRgwwMS33npr3vuC3Pj1lu26ySeffLKTa9KkiYnPO+88J/fvf//7/9u7d9CqkjAAwHPDlmoQRSTNFgG10G5tIoJoo4WFaOGjt7BUBBECaiEIBgsbH1iI2IgPBIt0CorgI1oZUUFdRLBYUEFEMXC3m505673eHO4jY76v+n/+Q84PGc69GSb/ifGvZupeuHAhxmbEzV/pM2rhwoVZrbqGvn371pee6J3Xr19n+fj4eIzXrVuX1dJ5qOmcyhBCOHz4cMt7rFixIsvXrl0b41OnTmW1dJbdxMREVpuenm55D+auDRs2/DT+leq85Tt37nSnIQZmaCg/h7F3794YV2e4X79+Pcbp+wFCCGHVqlUxTt8VEcL/Zx62e7dAWnv+/HlWu3z5coyPHz+e1ar3pDyjo6NZPjw83PLamzdvZvnMzExPemJw0mdTOgs1hPZzdKemprJ8//79Mf7x40eXumOu+PLlS5anz5Hqutm5c2eMV69endVGRka63tv9+/ezPH3vyfnz57Oa9+OUZ2xsLMvT7y/pd6IQQrh3715feuo2J3UBAAAAAApiUxcAAAAAoCBzbvzCjRs3YvzgwYOsNjk5GePqUfy6qkfqnz59GuMzZ8505R4M1qdPn2J89uzZltcdPHiwH+3wG1u6dGmMq/+q+uzZsyy/evVqX3qif96+fRvj6viF9PNk3759WW3Lli0/vS6EEI4dO5blS5YsaXn/W7duxfjcuXO/bpiiHT16NMZHjhwZXCP0TPqdePPmzVmtOiohtW3bthhXx72ko1iqn1PV50Y6KiHtpao6durr168tr6V8y5Yta1tPf/+nT5/udTsMWDrmrjoWqp0TJ05kuZEL89fFixdb5suXL89qCxYsyPJ0FNHt27ezWjqy7OXLl1nt8ePHMX737l1W+/79eydtU4h2IxE/fvzY73Z6wkldAAAAAICC2NQFAAAAACiITV0AAAAAgII0qvO02l7caHR+Mb021Ww2/xp0E52wbuaOZrPZGHQPnSh1zaQzudesWZPVDh06lOUnT57sS09d4FnTBcPDwzFeuXJlVhsfH49xOl83hBAmJiZa/sxr165l+ZMnT2I8MzNTq88usm6ow7qhDuumj65cuZLl27dvz/L0nShjY2N96akm66aGRYsWZfmbN29ivHjx4qzWaPz3Z8fdu3ez2saNG7N8Dnxv6ZR1w6z5G3xwDhw4kOXr16+P8e7du7PaHHsnQMfPGid1AQAAAAAKYlMXAAAAAKAgfwy6AYDfxfT0dIyr4xeY3z5//hzjhw8fZrWtW7f2ux0AqGXHjh1ZXh3ll46i4vezadOmLK+OXEilIxd27dqV1QoatwAUrDrKrt1ou1I5qQsAAAAAUBCbugAAAAAABbGpCwAAAABQEDN1AbpkcnIyxqOjo1nt0aNH/W4HAKCrhoacCZrP0vdHhBDChw8fYvzq1austmfPnhi/f/++t40BzFM+lQEAAAAACmJTFwAAAACgIMYvAHTJpUuXfhoDAEDpXrx4keUjIyMD6gSAEJzUBQAAAAAoik1dAAAAAICC2NQFAAAAACjIbGfq/hNC+LsXjTBrfw66gVmwbuYGa4Y6rBvqsG6ow7qhDuuGOqwb6rBumC1rhjo6XjeNZrPZy0YAAAAAAOgi4xcAAAAAAApiUxcAAAAAoCA2dQEAAAAACmJTFwAAAACgIDZ1AQAAAAAKYlMXAAAAAKAgNnUBAAAAAApiUxcAAAAAoCA2dQEAAAAACvIvxCrw4YrZOEwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1800x288 with 20 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def visualization(w1, b1, w2, b2):\n",
    "    x = test_x[:20]\n",
    "    x = x.reshape(x.shape[0], -1)\n",
    "    y = test_y[:20]\n",
    "    \n",
    "    # Forward Pass\n",
    "    h1 = np.dot(x, w1) + b1\n",
    "    a1 = sigmoid(h1)\n",
    "    h2 = np.dot(a1, w2) + b2\n",
    "    a2 = softmax(h2)\n",
    "    out = a2    \n",
    "    pred = np.argmax(out, axis=1)\n",
    "    \n",
    "    fig = plt.figure(figsize=(25, 4))\n",
    "    for index in np.arange(20):\n",
    "        ax = fig.add_subplot(2, 20/2, index+1, xticks=[], yticks=[])\n",
    "        ax.imshow(test_x[index], cmap='gray')\n",
    "        ax.set_title(\"{} ({})\".format(str(pred[index]), str(y[index][0])), \n",
    "                     color=(\"green\" if pred[index] == y[index] else \"red\"))\n",
    "        \n",
    "visualization(w1, b1, w2, b2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}