Week12 - ANN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Artificial Neural Networks(ANN)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activation functions\n",
    "\n",
    "**Sigmoid** \n",
    "$$\n",
    "    g(z) = \\frac{1}{1+e^{-z}}\n",
    "$$\n",
    "\n",
    "**ReLU**\n",
    "$$\n",
    "    g(z) = \\begin{cases}\n",
    "                z, \\space z \\ge 0 \\\\\n",
    "                0, \\space z < 0\n",
    "        \\end{cases}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "seed = 42\n",
    "rng = np.random.default_rng(seed==seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def grad_sigmoid(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.where(z >= 0, z, 0)\n",
    "\n",
    "def grad_relu(z):\n",
    "    return np.where(z >= 0, 1, 0)\n",
    "\n",
    "# A dictionary of activation functions will be used while inititalising the network\n",
    "hidden_act = {'sigmoid': sigmoid, 'relu': relu}\n",
    "grad_hidden_act = {'sigmoid': grad_sigmoid, 'relu': grad_relu}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output layer\n",
    "\n",
    "**Identity**\n",
    "\n",
    "$$\n",
    "g(z) = z\n",
    "$$\n",
    "\n",
    "**Softmax**\n",
    "\n",
    "$$\n",
    "g(Z) =  \\left[ {\\begin{array}{cc}\n",
    "                    ... & ... & ...\\\\\n",
    "                    ... & \\frac{exp(Z_{ij})}{\\sum_{j=1}^k  exp(Z_{ij})} & ... \\\\\n",
    "                    ... & ... & ... \\\\\n",
    "                \\end{array} } \\right]\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity(z):\n",
    "    return z\n",
    "\n",
    "def softmax(z):\n",
    "    '''Row-wise softmax'''\n",
    "    #Check if z is a matrix\n",
    "    assert z.ndim == 2\n",
    "\n",
    "    #To prevent overflow, subtract max, row-wise\n",
    "    z -= z.max(axis=1, keepdims=True)\n",
    "\n",
    "    #Compute row-wise softmax\n",
    "    prob = np.exp(z) / np.exp(z).sum(axis = 1, keepdims = True)\n",
    "\n",
    "    # Check if each row is a probability distribution\n",
    "    assert np.allclose(prob.sum(axis=1), np.ones(z.shape[0]))\n",
    "    return prob\n",
    "\n",
    "output_act = {'softmax': softmax, 'identity': identity}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss\n",
    "\n",
    "**Least Square**\n",
    "$$\n",
    "    L(y, \\hat{y}) = \\frac{1}{2}.(\\hat{y} - y)^T.(\\hat{y} - y)\n",
    "$$\n",
    "\n",
    "**Categorical Cross Entropy**\n",
    "$$\n",
    "    L(Y, \\hat{Y}) = -\\bf{1}_n^T \\space (Y \\odot log\\hat{Y}) \\space \\bf{1}_k\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def least_square(y, y_hat):\n",
    "    return 0.5 * np.sum((y_hat - y) * (y_hat - y))\n",
    "\n",
    "def cce(Y, Y_hat):\n",
    "    return -np.sum(Y * np.log(Y_hat))\n",
    "\n",
    "losses = {'least_square': least_square, 'cce': cce}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization\n",
    "\n",
    "**Network Architecture**\n",
    "* number of layers - indexed by `l`\n",
    "    * $ l = 0 $: Input layer\n",
    "    * $ 1 \\le l \\le L -1 $: Hidden layers\n",
    "    * $l = L$: Output layer\n",
    "* number of neurons per layer  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Helper function to compute total number of parameters in the network\n",
    "def count_params(layers):\n",
    "    num_params = 0\n",
    "    for l in range(1, len(layers)):\n",
    "        num_weights = layers[l-1] * layers[l]\n",
    "        num_biases = layers[l]\n",
    "        num_params += (num_weights + num_biases)\n",
    "\n",
    "    return num_params\n",
    "\n",
    "#Test\n",
    "assert count_params([64, 5, 10]) == (64 * 5 + 5) + (5 * 10 + 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parameter Initilisation**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cv2 import EMD\n",
    "\n",
    "\n",
    "def init_params(layers):\n",
    "    num_params = count_params(layers) # Number of parameters in the network\n",
    "\n",
    "    W = [None for _ in range(len(layers))] # weights\n",
    "    b = [None for _ in range(len(layers))] # biases\n",
    "    gW = [None for _ in range(len(layers))] # gradient of loss wrt weights\n",
    "    gb = [None for _ in range(len(layers))] # grad loss wrt biases\n",
    "\n",
    "    # Sample from N(0,1) to initialise the parameters\n",
    "    theta = rng. standard_normal(num_params) # master params\n",
    "    gtheta = np.zeros(num_params) # master gradients\n",
    "    \n",
    "    #(start, end) specify the portion of the theta that corresponds to the\n",
    "    #parameter, W_l or b_l\n",
    "    start, end = 0, 0\n",
    "    for l in range(1, len(layers)):\n",
    "        #reshape the section (start, end) an assign it to W[l]\n",
    "        end = start + layers[l-1] * layers[l]\n",
    "        W[l] = theta[start: end].reshape(layers[l-1], layers[l])\n",
    "        gW[l] = gtheta[start: end].reshape(layers[l-1], layers[l])\n",
    "\n",
    "        #reshape the section(start, end) and assign it to b[l]\n",
    "        start, end = end, end + layers[l]\n",
    "        b[l] = theta[start: end].reshape(layers[l])\n",
    "        gb[l] = gtheta[start: end].reshape(layers[l])\n",
    "        start = end\n",
    "\n",
    "    return theta, gtheta, W, b, gW, gb\n",
    "\n",
    "## test init_params\n",
    "layers = [64, 32, 10]\n",
    "params = init_params([6, 32, 10])\n",
    "for l in range(1, len(layers)):\n",
    "    #Check if the weights are view of the master vector\n",
    "    assert params[2][l].base is params[0]\n",
    "    assert params[3][l].base is params[0]\n",
    "    assert params[4][l].base is params[1]\n",
    "    assert params[5][l].base is params[1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Forward pass\n",
    "\n",
    "The forward pass algorithm is as follows. First, we initalise $A_0 = X$. The, we iteratively compute pre-activations and activations for each layer $l$ using the equations givenbelow:\n",
    "$$\n",
    "    Z_l = A_{l-1}W_l + b_l\n",
    "    A_l = g(Z_l)\n",
    "$$\n",
    "\n",
    "Finally, the netwrk's output is given by $\\hat{y} = A_l$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backward pass\n",
    "\n",
    "For backward pass, we first initialise the gradients of the  pre activations at layer $L$ as $Z_L^{(g)} = \\hat{Y} - Y $. The other gradients can then be iteratively updated.\n",
    "\n",
    "$$\n",
    "    W_l^{(g)} = A_{l-1}^T Z_l^{(g)} \\\\\n",
    "$$\n",
    "$$\n",
    "    b_l^{(g)} = Z_l^{(g)^T} \\bf{1_n} \\\\\n",
    "$$\n",
    "$$\n",
    "    A_{l-1}^{(g)} = Z_l^{(g)} W_l^T \\\\\n",
    "$$\n",
    "$$\n",
    "    Z_{l-1}^{(g)} = A_{l-1}^{(g)} \\odot g'(Z_{l-1})\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network:\n",
    "\n",
    "    def __init__(self, layers, activation_choice='relu',\n",
    "                output_choice='softmax', loss_choice='cce'):\n",
    "        \n",
    "        self.layers = layers\n",
    "        # Parameters and gradients\n",
    "        self.theta, self.gtheta, self.W, self.b, self.gW, self.gb = init_params(layers)\n",
    "\n",
    "        #Activation functions\n",
    "        self.ghid = hidden_act[activation_choice]\n",
    "        self.grad_ghid = grad_hidden_act[activation_choice]\n",
    "        self.gout = output_act[output_choice]\n",
    "\n",
    "        #Loss\n",
    "        self.loss = losses[loss_choice]\n",
    "\n",
    "    def forward(self, X):\n",
    "        self.Z = [None for _ in range(len(self.layers))]\n",
    "        self.A = [None for _ in range(len(self.layers))]\n",
    "        self.A[0] = X\n",
    "        self.Z[0] = X\n",
    "\n",
    "        for l in range(1, len(self.layers)):\n",
    "            self.Z[l] = self.A[l-1] @ self.W[l] + self.b[l]\n",
    "            self.A[l] = self.ghid(self.Z[l])\n",
    "        \n",
    "        self.A[-1] = self.gout(self.Z[-1])\n",
    "        return self.A[-1]\n",
    "        \n",
    "    def backward(self, Y, Y_hat):\n",
    "        gZ = [None for _ in range(len(self.layers))]\n",
    "        gA = [None for _ in range(len(self.layers))]\n",
    "        gZ[-1] = Y_hat -Y\n",
    "\n",
    "        for l in range(len(self.layers) - 1, 0, -1):\n",
    "            self.gW[l][:,:] = self.A[l-1].T @ gZ[l]\n",
    "            self.gb[l][:] = np.sum(gZ[l].T, axis = 1)\n",
    "\n",
    "            gA[l-1] = gZ[l] @ self.W[l].T\n",
    "            gZ[l-1] = gA[l-1] * self.grad_ghid(self.Z[l-1])\n",
    "\n",
    "    def fit(self, X, Y,\n",
    "                lr = 0.01, epochs = 100, batch_size = 100):\n",
    "\n",
    "        self.losses = []\n",
    "\n",
    "        for epoch in range(epochs):\n",
    "            #Compute the loss\n",
    "            Y_hat = self.forward(X)\n",
    "            self.losses.append(self.loss(Y, Y_hat))\n",
    "            #Shuffle the dataset\n",
    "            indices = np.arange(X.shape[0])\n",
    "            rng.shuffle(indices)\n",
    "            X, Y = X[indices], Y[indices]\n",
    "            #number of batches\n",
    "            num_batches = X.shape[0] // batch_size\n",
    "            #Mini-batch GD\n",
    "            for b in range(num_batches):\n",
    "                Xb = X[b * batch_size: (b + 1) * batch_size]\n",
    "                Yb = Y[b * batch_size: (b + 1) * batch_size]\n",
    "                #Compute the prediction for this batch\n",
    "                Y_hat_b =  self.forward(Xb)\n",
    "                #Compute the gradients for this batch\n",
    "                self.backward(Yb, Y_hat_b)\n",
    "                #Update the gradients of all parameters\n",
    "                self.theta -= lr*self.gtheta\n",
    "\n",
    "    def predict(self, X):\n",
    "        Y_hat = self.forward(X)\n",
    "        if X.shape[-1] == 1: # regression\n",
    "            return Y_hat\n",
    "\n",
    "        else: #classification\n",
    "            return np.argmax(Y_hat, axis = 1)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Demonstration\n",
    "\n",
    "We are using digits dataset from sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample image with label 0\n",
      "(1797, 8, 8)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAALGUlEQVR4nO3d/6uW9R3H8ddrR81Vplu2Co8sGSXEYlnOIUYwpWErKthYCjUWA2FQFMmiRmPbPxDuhxGI1YJc0qwgWl8Wq2iBM7/kKr8Nk4ZHKo2+C6kn3/vh3ILFsXPd97muz3Wf954PkM6Xm/vzvrGn132uc9/XxxEhAHl8re0BANSLqIFkiBpIhqiBZIgaSGZSE3c6xafEVJ3WxF23anhm2cd0zjnvF1tr/6EZxdaaOnS02FpxdLjYWiV9pkM6Eoc92vcaiXqqTtMPvKSJu27Vez9ZWHS9X69cV2yt3265tthaF9z+drG1ht95t9haJW2Mf5z0ezz9BpIhaiAZogaSIWogGaIGkiFqIBmiBpIhaiAZogaSqRS17aW2d9veY/vOpocC0Lsxo7Y9IOlPkq6UdKGk5bYvbHowAL2pcqReIGlPROyNiCOS1kkq90JhAF2pEvUsSftO+Hyo87UvsL3C9mbbm4/qcF3zAehSbSfKImJ1RMyPiPmTdUpddwugS1Wi3i9p9gmfD3a+BqAPVYl6k6Tzbc+xPUXSMklPNDsWgF6NeZGEiBi2fbOkZyUNSLo/IrY3PhmAnlS68klEPCXpqYZnAVADXlEGJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJNPIDh1ZldwxQ5KWTfug2FqrZnxabK2/bX222FqX/v5XxdaSpJmrNxRdbzQcqYFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSKbKDh332z5g+40SAwEYnypH6j9LWtrwHABqMmbUEfGSpPcLzAKgBrW9S8v2CkkrJGmqTq3rbgF0iW13gGQ4+w0kQ9RAMlV+pfWwpA2S5toesv3L5scC0Ksqe2ktLzEIgHrw9BtIhqiBZIgaSIaogWSIGkiGqIFkiBpIZsJvuzO8+NJiay2btq3YWpJ05dJlxdaa/tquYmv97OUlxdZ6f97nxdaSpJlFVxsdR2ogGaIGkiFqIBmiBpIhaiAZogaSIWogGaIGkiFqIBmiBpKpco2y2bZfsL3D9nbbt5YYDEBvqrz2e1jSyojYanuapC22n4uIHQ3PBqAHVbbdeTsitnY+/kTSTkmzmh4MQG+6epeW7fMkzZO0cZTvse0O0AcqnyizfbqkRyXdFhEff/n7bLsD9IdKUduerJGg10bEY82OBGA8qpz9tqT7JO2MiHuaHwnAeFQ5Ui+SdKOkxba3df78uOG5APSoyrY7L0tygVkA1IBXlAHJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQzITfS+uzM8s9hLsPXFRsLUk6VnB/q5I2vf6dtkdIjSM1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8kQNZBMlQsPTrX9iu1/d7bd+UOJwQD0psprLA9LWhwRn3YuFfyy7acj4l8NzwagB1UuPBiSPu18OrnzJ5ocCkDvql7Mf8D2NkkHJD0XEaNuu2N7s+3NR3W45jEBVFUp6oj4PCIuljQoaYHt745yG7bdAfpAV2e/I+JDSS9IWtrINADGrcrZ77Nsz+h8/HVJV0jK+UZfIIEqZ7/PlfSg7QGN/CPwSEQ82exYAHpV5ez3axrZkxrABMAryoBkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIZuJvu/ONcv8urd2wsNhaknSBXim6XimTph8pttbwR1OKrdUvOFIDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8kQNZBM5ag7F/R/1TYXHQT6WDdH6lsl7WxqEAD1qLrtzqCkqyStaXYcAONV9Ui9StIdko6d7AbspQX0hyo7dFwt6UBEbPmq27GXFtAfqhypF0m6xvZbktZJWmz7oUanAtCzMaOOiLsiYjAizpO0TNLzEXFD45MB6Am/pwaS6epyRhHxoqQXG5kEQC04UgPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJTPhtd6Z+cNL3mNTu+xe9WWwtSfqo4FqTzjm72FrXX/iVbyOo1SNPX1ZsrX7BkRpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWQqvUy0cyXRTyR9Lmk4IuY3ORSA3nXz2u8fRsR7jU0CoBY8/QaSqRp1SPq77S22V4x2A7bdAfpD1affl0XEftvfkvSc7V0R8dKJN4iI1ZJWS9IZ/mbUPCeAiiodqSNif+e/ByQ9LmlBk0MB6F2VDfJOsz3t+MeSfiTpjaYHA9CbKk+/z5b0uO3jt/9LRDzT6FQAejZm1BGxV9L3CswCoAb8SgtIhqiBZIgaSIaogWSIGkiGqIFkiBpIZsJvu3PG7nKb0/xu8Mlia0nSz1fcXmytydcdLLZWSXPu2tD2CMVxpAaSIWogGaIGkiFqIBmiBpIhaiAZogaSIWogGaIGkiFqIJlKUdueYXu97V22d9pe2PRgAHpT9bXff5T0TET81PYUSac2OBOAcRgzatvTJV0u6ReSFBFHJB1pdiwAvary9HuOpIOSHrD9qu01net/fwHb7gD9oUrUkyRdIuneiJgn6ZCkO798o4hYHRHzI2L+ZJ1S85gAqqoS9ZCkoYjY2Pl8vUYiB9CHxow6It6RtM/23M6Xlkja0ehUAHpW9ez3LZLWds5875V0U3MjARiPSlFHxDZJ85sdBUAdeEUZkAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8lM+L20jr22q9ha19+7sthaknT3yoeLrbXqzSXF1tp08UCxtf4fcaQGkiFqIBmiBpIhaiAZogaSIWogGaIGkiFqIBmiBpIZM2rbc21vO+HPx7ZvKzAbgB6M+TLRiNgt6WJJsj0gab+kx5sdC0Cvun36vUTSmxHx3yaGATB+3b6hY5mkUd9lYHuFpBWSNJX984DWVD5Sd675fY2kv472fbbdAfpDN0+/r5S0NSLebWoYAOPXTdTLdZKn3gD6R6WoO1vXXiHpsWbHATBeVbfdOSTpzIZnAVADXlEGJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDKOiPrv1D4oqdu3Z86U9F7tw/SHrI+Nx9Web0fEWaN9o5Goe2F7c0TMb3uOJmR9bDyu/sTTbyAZogaS6aeoV7c9QIOyPjYeVx/qm5+pAdSjn47UAGpA1EAyfRG17aW2d9veY/vOtuepg+3Ztl+wvcP2dtu3tj1TnWwP2H7V9pNtz1In2zNsr7e9y/ZO2wvbnqlbrf9M3dkg4D8auVzSkKRNkpZHxI5WBxsn2+dKOjcittqeJmmLpOsm+uM6zvbtkuZLOiMirm57nrrYflDSPyNiTecKuqdGxIctj9WVfjhSL5C0JyL2RsQRSeskXdvyTOMWEW9HxNbOx59I2ilpVrtT1cP2oKSrJK1pe5Y62Z4u6XJJ90lSRByZaEFL/RH1LEn7Tvh8SEn+5z/O9nmS5kna2PIodVkl6Q5Jx1qeo25zJB2U9EDnR4s1nYtuTij9EHVqtk+X9Kik2yLi47bnGS/bV0s6EBFb2p6lAZMkXSLp3oiYJ+mQpAl3jqcfot4vafYJnw92vjbh2Z6skaDXRkSWyysvknSN7bc08qPSYtsPtTtSbYYkDUXE8WdU6zUS+YTSD1FvknS+7TmdExPLJD3R8kzjZtsa+dlsZ0Tc0/Y8dYmIuyJiMCLO08jf1fMRcUPLY9UiIt6RtM/23M6XlkiacCc2u90gr3YRMWz7ZknPShqQdH9EbG95rDosknSjpNdtb+t87TcR8VR7I6GCWySt7Rxg9kq6qeV5utb6r7QA1Ksfnn4DqBFRA8kQNZAMUQPJEDWQDFEDyRA1kMz/ACA9oV03rwJJAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from unicodedata import digit\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_digits\n",
    "digits = load_digits()\n",
    "\n",
    "X = digits.images\n",
    "#Normalising the data\n",
    "X /= np.max(X)\n",
    "y = digits.target\n",
    "plt.imshow(X[0])\n",
    "print(f'Sample image with label {y[0]}')\n",
    "print(X.shape)\n",
    "\n",
    "#Reshape input\n",
    "X = X.reshape(-1, 64)\n",
    "#Input size\n",
    "isize = X.shape[-1]\n",
    "#output size\n",
    "osize = len(np.unique(y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehot_encoder(y):\n",
    "    k = len(np.unique(y))\n",
    "    return np.eye(k)[y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data, shape: (1078, 64) (1078, 10)\n",
      "Test data, shape: (719, 64) (719, 10)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=seed)\n",
    "Y_train = onehot_encoder(y_train)\n",
    "Y_test = onehot_encoder(y_test)\n",
    "\n",
    "print('Training data, shape:', X_train.shape, Y_train.shape)\n",
    "print('Test data, shape:', X_test.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Applying ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test-data size = 719\n",
      "Accuracy =  0.97\n",
      "Number of parameters = 2410\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAHwCAYAAAA/wLxAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxUklEQVR4nO3deZQcZ33u8edX1dUz2hdrJGvDsiMBMcQYEMY2WcBmsdnMFpZAbIhvfJJLAmS7gST3OiFwLhAOi0nCOQ4YzOo4BoIDXIxjTAgxXmQbbCTjSHiTZC1jS7LWmenld/+ot0et8cxImq6e7lf9/Zz06aq3qqvfKiLP0+9SZe4uAADQe5JOVwAAAHQGIQAAgB5FCAAAoEcRAgAA6FGEAAAAehQhAACAHkUIANATzOztZvajTtcD6CaEACBSZvaQmb240/WYCjN7oZnVzWz/mNc5na4b0EtKna4AgJ71qLuv6HQlgF5GSwBwgjGzPjP7hJk9Gl6fMLO+sG2RmX3LzPaY2S4z+08zS8K2PzezrWa2z8zuN7Pzxzn2881su5mlTWWvNbN7wvJZZrbOzPaa2Q4z+9gUz+EHZvZ/zez2cKxvmtnCpu2vNrP14Tx+YGa/3LRtpZl93cwGzexxM/v7Mcf+qJntNrMHzezCpvK3m9kD4fwfNLO3TqXuQEwIAcCJ5y8lnS3pTEnPknSWpL8K2/5E0hZJA5KWSPoLSW5mT5P0B5Ke5+5zJL1M0kNjD+zut0k6IOm8puLfkvSVsPxJSZ9097mSfknStS2cx8WSfkfSUklVSVdIkpk9VdJXJb0nnMd3JP2bmZVDOPmWpIclrZK0XNI1Tcd8vqT7JS2S9BFJn7XcrHD8C8P5nyvpJy3UHYgCIQA48bxV0vvdfae7D0r6G0m/HbZVlP9RPcXdK+7+n54/QKQmqU/S6WaWuftD7v6LCY7/VUlvkSQzmyPp5aGscfzVZrbI3fe7+62T1HNZ+CXf/JrVtP2L7v4zdz8g6X9LemP4I/8mSd929xvdvSLpo5JmKP/DfZakZZL+zN0PuPuQuzcPBnzY3f/J3WuSrg7XYknYVpf0TDOb4e7b3H39JHUHTgiEAODEs0z5L+GGh0OZJP2dpE2Svheavt8rSe6+Sfkv67+WtNPMrjGzZRrfVyS9LnQxvE7SXe7e+L5LJT1V0s/N7A4ze+Uk9XzU3eePeR1o2r55zDlkyn/BH3F+7l4P+y6XtFL5H/rqBN+5velzB8Pi7PC9b5L0e5K2mdm3zezpk9QdOCEQAoATz6OSTmlaf0ook7vvc/c/cffTJL1a0h83+v7d/Svu/qvhsy7pw+Md3N03KP8jfKGO7AqQu29097dIWhw+f92YX/fHY+WYc6hIemzs+ZmZhX23Kg8DTzGz4x707O43uPtLlLcO/FzSP02x3kA0CAFA3DIz6296lZQ3zf+VmQ2Y2SJJ/0fSlyTJzF5pZqvDH84nlHcD1M3saWZ2Xvh1PyTpkPLm8Yl8RdK7Jf26pH9pFJrZ28xsIPw63xOKJzvOZN5mZqeb2UxJ75d0XWjGv1bSK8zsfDPLlI9zGJZ0i6TbJW2T9CEzmxWuyQuO9kVmtsTMLgqBZVjS/hbqDUSDEADE7TvK/2A3Xn8t6QOS1km6R9K9ku4KZZK0RtK/K/8j92NJ/+juNysfD/Ah5b+0tyv/Jf++Sb73q5J+Q9L33f2xpvILJK03s/3KBwm+2d0PTXCMZePcJ+D1Tdu/KOnzoT79kt4lSe5+v6S3SfpUqO+rJL3K3UdCSHiVpNWSHlE+CPJNk5xHQyLpj5W3MuwK5/b7x/A5IGqWjwkCgO5hZj+Q9CV3/0yn6wKcyGgJAACgRxECAADoUXQHAADQo2gJAACgRxECAADoUT33FMFFixb5qlWrOl0NAACmxZ133vmYuw+Mt63nQsCqVau0bt26TlcDAIBpYWYPT7SN7gAAAHoUIQAAgB5FCAAAoEcRAgAA6FGEAAAAehQhAACAHkUIAACgRxECAADoUYQAAAB6FCEAAIAeRQgAAKBHEQIAAOhRhAAAAHoUIQAAgB5FCAAAoEcRAgAA6FGEgBbsG6rowHC109UAAGBKCAEteMnHfqj3/9uGTlcDAIApIQS0ICuZRmr1TlcDAIApIQS0IEsTQgAAIFqEgBaU00SVKiEAABAnQkALyiVaAgAA8SIEtCBLE1UIAQCASBECWpB3B3inqwEAwJQQAlqQ0R0AAIgYIaAF5dQ0wsBAAECkCAEtKJcYEwAAiBchoAUMDAQAxIwQ0IIsTegOAABEixDQgvyOgcwOAADEiRDQgj7GBAAAIkYIaEHG7AAAQMQIAS1gYCAAIGaEgBaUS4mqdVe9zrgAAEB8CAEtyNL88nHXQABAjAgBLSiHEECXAAAgRoSAFpRLjRBAdwAAID6EgBZktAQAACJGCGhBlpokMU0QABAlQkALGt0BDAwEAMSIENACBgYCAGJGCGjB6BRBugMAABEiBLQgK9ESAACIFyGgBeXRlgCmCAIA4kMIaEG5FGYH0BIAAIgQIaAFo/cJYEwAACBChIAWlBkTAACIGCGgBTxACAAQM0JAC8pMEQQARIwQ0AIeIAQAiFnbQoCZXWVmO83sZ01lC83sRjPbGN4XhHIzsyvMbJOZ3WNmz2n6zCVh/41mdklT+XPN7N7wmSvMzNp1LhPhAUIAgJi1syXg85IuGFP2Xkk3ufsaSTeFdUm6UNKa8LpM0qelPDRIulzS8yWdJenyRnAI+/xu0+fGflfb8QAhAEDM2hYC3P2HknaNKb5I0tVh+WpJr2kq/4LnbpU038yWSnqZpBvdfZe775Z0o6QLwra57n6ru7ukLzQda9rwACEAQMyme0zAEnffFpa3S1oSlpdL2ty035ZQNln5lnHKp1WW0B0AAIhXxwYGhl/w0zKizswuM7N1ZrZucHCwsOMmiamUGN0BAIAoTXcI2BGa8hXed4byrZJWNu23IpRNVr5inPJxufuV7r7W3dcODAy0fBLNsjShJQAAEKXpDgHXS2qM8L9E0jebyi8OswTOlvRE6Da4QdJLzWxBGBD4Ukk3hG17zezsMCvg4qZjTatyKWGKIAAgSqV2HdjMvirphZIWmdkW5aP8PyTpWjO7VNLDkt4Ydv+OpJdL2iTpoKR3SJK77zKzv5V0R9jv/e7eGGz4P5XPQJgh6f+F17TL0kTDdAcAACLUthDg7m+ZYNP54+zrkt45wXGuknTVOOXrJD2zlToWoZwa3QEAgChxx8AW5d0BhAAAQHwIAS3K0oTZAQCAKBECWsTsAABArAgBLSqXEo0wOwAAECFCQIvKaaIK3QEAgAgRAlqUlYxnBwAAokQIaFGZMQEAgEgRAlrE7AAAQKwIAS3KSgndAQCAKBECWkR3AAAgVoSAFuWzA5giCACIDyGgRcwOAADEihDQooz7BAAAIkUIaFGZgYEAgEgRAlpUTvMQkD8NGQCAeBACWpSlidylWp0QAACICyGgReVSfgkrPEQIABAZQkCLsjS/hIwLAADEhhDQonJqksStgwEA0SEEtOhwdwAhAAAQF0JAixrdAYQAAEBsCAEtGh0TQHcAACAyhIAWMTAQABArQkCL+pgiCACIFCGgRXQHAABiRQhoURamCDIwEAAQG0JAixpTBBkTAACIDSGgRXQHAABiRQhoETcLAgDEihDQojI3CwIARIoQ0KKs0RJQZYogACAuhIAWNWYHDNMSAACIDCGgRX1pKkmqMDAQABAZQkCLshL3CQAAxIkQ0CKmCAIAYkUIaFEpoSUAABAnQkCLzEzlUqIRHiAEAIgMIaAA5TShOwAAEB1CQAGy1OgOAABEhxBQgHIpIQQAAKJDCChARncAACBChIAClNOERwkDAKJDCCgA3QEAgBgRAgqQpYkqTBEEAESGEFCALDXGBAAAokMIKEB+syBCAAAgLoSAAuTdAYQAAEBcCAEF4I6BAIAYEQIKQEsAACBGhIAC5FMEmR0AAIgLIaAA3DEQABAjQkAByiVjdgAAIDqEgAKUGRMAAIgQIaAAdAcAAGJECChAxrMDAAARIgQUoByeHeDODAEAQDwIAQUol/LLyDRBAEBMCAEFyFKTJGYIAACiQggoQDkNLQEMDgQARIQQUIBstDuAEAAAiAchoABZaAkYpiUAABARQkABRrsDaAkAAESEEFAAZgcAAGJECChAozuAuwYCAGLSkRBgZn9kZuvN7Gdm9lUz6zezU83sNjPbZGb/bGblsG9fWN8Utq9qOs77Qvn9ZvayTpyLxBRBAECcpj0EmNlySe+StNbdnykplfRmSR+W9HF3Xy1pt6RLw0culbQ7lH887CczOz187hmSLpD0j2aWTue5NJSZHQAAiFCnugNKkmaYWUnSTEnbJJ0n6bqw/WpJrwnLF4V1he3nm5mF8mvcfdjdH5S0SdJZ01P9I5XpDgAARGjaQ4C7b5X0UUmPKP/j/4SkOyXtcfdq2G2LpOVhebmkzeGz1bD/Sc3l43zmCGZ2mZmtM7N1g4ODxZ6QDo8JoCUAABCTTnQHLFD+K/5UScskzVLenN827n6lu69197UDAwOFH5/uAABAjDrRHfBiSQ+6+6C7VyR9XdILJM0P3QOStELS1rC8VdJKSQrb50l6vLl8nM9Mq9HZAUwRBABEpBMh4BFJZ5vZzNC3f76kDZJulvSGsM8lkr4Zlq8P6wrbv+/5M3uvl/TmMHvgVElrJN0+TedwBMYEAABiVDr6LsVy99vM7DpJd0mqSrpb0pWSvi3pGjP7QCj7bPjIZyV90cw2SdqlfEaA3H29mV2rPEBUJb3T3WvTejIB3QEAgBhNewiQJHe/XNLlY4of0Dij+919SNJvTnCcD0r6YOEVPE6N+wQQAgAAMeGOgQVoPEWQ7gAAQEwIAQUYHRNASwAAICKEgAKM3iegyuwAAEA8CAEFSBNTmphGah0ZlwgAwJQQAgqSpcajhAEAUSEEFKScJgwMBABEhRBQkHIpYWAgACAqhICCZGmiCi0BAICIEAIKUi4l3CwIABAVQkBBsjRhYCAAICqEgIJkaaJhugMAABEhBBSE7gAAQGwIAQUpp0YIAABEhRBQkIz7BAAAIkMIKEg+MJAQAACIByGgIPnNgpgdAACIByGgIPltg3mAEAAgHoSAgvAAIQBAbAgBBWGKIAAgNoSAgjA7AAAQG0JAQbKUpwgCAOJCCChIH90BAIDIEAIKwgOEAACxIQQUJEsT1equWp0gAACIAyGgIOVSfinpEgAAxIIQUJAsNUlicCAAIBqEgII0WgKYJggAiAUhoCBZSncAACAuhICClBshoMrAQABAHAgBBcka3QE1HiIEAIgDIaAg5cbAQFoCAACRIAQUhCmCAIDYEAIK0hgYyBRBAEAsCAEFGZ0dwBRBAEAkCAEFGb1PAC0BAIBIEAIKMjpFkIcIAQAiQQgoyOiYALoDAACRIAQUhNkBAIDYEAIKwgOEAACxIQQUpEx3AAAgMoSAgvAAIQBAbAgBBWFMAAAgNoSAgjA7AAAQG0JAQQ4PDOQ+AQCAOBACCmJmKqcJ3QEAgGgQAgqUpUZ3AAAgGoSAAmUlWgIAAPEgBBSI7gAAQEwIAQXK0kQjVQYGAgDiQAgoULmUcNtgAEA0CAEFKqeJKgwMBABEghBQoKxkjAkAAESDEFCgLKU7AAAQD0JAgfKBgYQAAEAcCAEF6uM+AQCAiBACCkR3AAAgJoSAAmWpqcJ9AgAAkSAEFKhcSukOAABEgxBQoCw1ugMAANEgBBSozOwAAEBECAEFKjM7AAAQEUJAgbI0UaXGwEAAQByOGgLM7JfMrC8sv9DM3mVm89teswhxsyAAQEyOpSXga5JqZrZa0pWSVkr6Sitfambzzew6M/u5md1nZueY2UIzu9HMNob3BWFfM7MrzGyTmd1jZs9pOs4lYf+NZnZJK3UqQuMpgu60BgAAut+xhIC6u1clvVbSp9z9zyQtbfF7Pynpu+7+dEnPknSfpPdKusnd10i6KaxL0oWS1oTXZZI+LUlmtlDS5ZKeL+ksSZc3gkOnlFOTJFXrhAAAQPc7lhBQMbO3SLpE0rdCWTbVLzSzeZJ+XdJnJcndR9x9j6SLJF0ddrta0mvC8kWSvuC5WyXNN7Olkl4m6UZ33+XuuyXdKOmCqdarCFmaX066BAAAMTiWEPAOSedI+qC7P2hmp0r6YgvfeaqkQUmfM7O7zewzZjZL0hJ33xb22S5pSVheLmlz0+e3hLKJyjumEQKYIQAAiMFRQ4C7b3D3d7n7V0Nz+xx3/3AL31mS9BxJn3b3Z0s6oMNN/43vdEmFtamb2WVmts7M1g0ODhZ12Ccpl0JLACEAABCBY5kd8AMzmxv64O+S9E9m9rEWvnOLpC3ufltYv055KNgRmvkV3neG7VuVD0ZsWBHKJip/Ene/0t3XuvvagYGBFqo+uTLdAQCAiBxLd8A8d98r6XXK++afL+nFU/1Cd98uabOZPS0UnS9pg6TrlY87UHj/Zli+XtLFYZbA2ZKeCN0GN0h6qZktCC0ULw1lHZOV8oGB3CsAABCD0rHsE36Zv1HSXxb0vX8o6ctmVpb0gPJxB4mka83sUkkPh++TpO9IermkTZIOhn3l7rvM7G8l3RH2e7+77yqoflNSTlNJjAkAAMThWELA+5X/wv4vd7/DzE6TtLGVL3X3n0haO86m88fZ1yW9c4LjXCXpqlbqUqQsTBGkOwAAEIOjhgB3/xdJ/9K0/oCk17ezUrHKGBgIAIjIsQwMXGFm3zCzneH1NTNbMR2Vi01fY4ogLQEAgAgcy8DAzykfnLcsvP4tlGGMRksAAwMBADE4lhAw4O6fc/dqeH1eUvvm2UVs9I6BtVqHawIAwNEdSwh43MzeZmZpeL1N0uPtrliMDt8ngJYAAED3O5YQ8DvKp+ttl7RN0hskvb2NdYpWefQ+AYwJAAB0v2OZHfCwpFc3l5nZRyX9absqFSseIAQAiMmxtASM541H36X38AAhAEBMphoCrNBanCDKJUIAACAeE3YHhAcGjbtJhIBxNVoChukOAABEYLIxAXcqf5zveH/wR9pTnbiVU+4TAACIx4QhwN1Pnc6KnAjoDgAAxGSqYwIwjjQxJUYIAADEgRBQsCxNmCIIAIgCIaBg5VLCUwQBAFGYMASY2XlNy6eO2fa6dlYqZuU0oTsAABCFyVoCPtq0/LUx2/6qDXU5IdAdAACIxWQhwCZYHm8dQbmUMEUQABCFyUKAT7A83jqCLDXGBAAAojDZzYJOM7Prlf/qbywrrHMPgQnQHQAAiMVkIeCipuWPjtk2dh1B3h1ACAAAdL/JQsAGSQPuvqG50MxOlzTY1lpFjNkBAIBYTDYm4FOSFo1TfpKkT7anOvGjOwAAEIvJQsBqd//h2EJ3/09JZ7SvSnHLSolGmB0AAIjAZCFgziTbsqIrcqIop4kqtAQAACIwWQjYZGYvH1toZhdKeqB9VYpbuWSMCQAARGGygYF/JOlbZvZGSXeGsrWSzpH0ynZXLFZZyrMDAABxmLAlwN3/W9KvSPoPSavC6z8knRG2YRx0BwAAYjFhS4CZrZa0xN0/N6b8BWa23d1/0fbaRYiBgQCAWEw2JuATkvaOU743bMM4ymmikWqt09UAAOCoJgsBS9z93rGFoWxV22oUOR4gBACIxWQhYP4k22YUXI8TRpYyOwAAEIfJQsA6M/vdsYVm9j90eLYAxsjSRNW6q16nNQAA0N0mmyL4HknfMLO36sgpgmVJr21zvaKVpXmuGqnV1Z+kHa4NAAATmzAEuPsOSeea2YskPTMUf9vdvz8tNYtUXykPAZVaXf0ZIQAA0L0mawmQJLn7zZJunoa6nBBGWwK4VwAAoMtNNiYAU9AIAcwQAAB0O0JAwcpN3QEAAHQzQkDBstQkiecHAAC6HiGgYGXGBAAAIkEIKBjdAQCAWBACCnZ4YCAhAADQ3QgBBWuEgGG6AwAAXY4QULDD3QFMEQQAdDdCQMEaAwMrtAQAALocIaBgWYkpggCAOBACCsbAQABALAgBBeM+AQCAWBACCtYYGEh3AACg2xECCpYxMBAAEAlCQMGYIggAiAUhoGA8QAgAEAtCQMGyhIGBAIA4EAIKliSmLDWmCAIAuh4hoA2yNCEEAAC6HiGgDbI0oTsAAND1CAFtUC4lGmF2AACgyxEC2qBMdwAAIAKEgDbIUqM7AADQ9QgBbcDAQABADAgBbVAuEQIAAN2PENAGWZpomO4AAECXIwS0AQMDAQAxIAS0Qd4dwBRBAEB361gIMLPUzO42s2+F9VPN7DYz22Rm/2xm5VDeF9Y3he2rmo7xvlB+v5m9rEOn8iTcNhgAEINOtgS8W9J9TesflvRxd18tabekS0P5pZJ2h/KPh/1kZqdLerOkZ0i6QNI/mlk6TXWfFHcMBADEoCMhwMxWSHqFpM+EdZN0nqTrwi5XS3pNWL4orCtsPz/sf5Gka9x92N0flLRJ0lnTcgJHkd8xkBAAAOhunWoJ+ISk/yWp8ZfyJEl73L0a1rdIWh6Wl0vaLElh+xNh/9HycT5zBDO7zMzWmdm6wcHBAk9jfAwMBADEYNpDgJm9UtJOd79zur7T3a9097XuvnZgYKDt30d3AAAgBqUOfOcLJL3azF4uqV/SXEmflDTfzErh1/4KSVvD/lslrZS0xcxKkuZJerypvKH5Mx3F7AAAQAymvSXA3d/n7ivcfZXygX3fd/e3SrpZ0hvCbpdI+mZYvj6sK2z/vrt7KH9zmD1wqqQ1km6fptOYVJYmqtASAADocp1oCZjIn0u6xsw+IOluSZ8N5Z+V9EUz2yRpl/LgIHdfb2bXStogqSrpne5em/5qP1lWMg0zJgAA0OU6GgLc/QeSfhCWH9A4o/vdfUjSb07w+Q9K+mD7ajg1jYGB7q58IgMAAN2HOwa2QTlN5C7V6owLAAB0L0JAG2Sl/LJyrwAAQDcjBLRBluaXtVKlJQAA0L0IAW1QpiUAABABQkAblNN8MCB3DQQAdDNCQBs0ugO4ayAAoJsRAtqg0R1ASwAAoJsRAtpgtCWAEAAA6GKEgDYo0x0AAIgAIaANDncHMEUQANC9CAFtMHqfALoDAABdjBDQBlmYIkh3AACgmxEC2oCBgQCAGBAC2qCPKYIAgAgQAtqAmwUBAGJACGiDjJYAAEAECAFtMHqfAKYIAgC6GCGgDcqjjxKmJQAA0L0IAW2QlcIUQboDAABdjBDQBrQEAABiQAhogzQxmTEwEADQ3QgBbWBmytJEw4QAAEAXIwS0SV+aqFJldgAAoHsRAtokKyV0BwAAuhohoE2y1LhjIACgqxEC2iRLaQkAAHQ3QkCblEsJ9wkAAHQ1QkCblNOE7gAAQFcjBLQJ3QEAgG5HCGiTcilRhQcIAQC6GCGgTbLUGBMAAOhqhIA2yRgTAADocoSANunjZkEAgC5HCGgTBgYCALodIaBN6A4AAHQ7QkCbMDsAANDtCAFtkqXcMRAA0N0IAW1S5gFCAIAuRwhoEwYGAgC6HSGgTcpMEQQAdDlCQJvkLQGuep3BgQCA7kQIaJNyKb+0lTqtAQCA7kQIaJNyGkIA0wQBAF2KENAmWWqSpAozBAAAXYoQ0CZZ6A7gXgEAgG5FCGiTRncA9woAAHQrQkCbjA4MpCUAANClCAFtkqV0BwAAuhshoE1GZwdUmR0AAOhOhIA2YWAgAKDbEQLapDFFkIGBAIBuRQhok8M3CyIEAAC6EyGgTZgdAADodoSANsm4TwAAoMsRAtqEKYIAgG5HCGiTvhIPEAIAdDdCQJtkDAwEAHQ5QkCbMEUQANDtCAFtwuwAAEC3IwS0CQMDAQDdjhDQJjxKGADQ7QgBbZIkplJidAcAALoWIaCNsjRhiiAAoGtNewgws5VmdrOZbTCz9Wb27lC+0MxuNLON4X1BKDczu8LMNpnZPWb2nKZjXRL232hml0z3uRxNlhrdAQCArtWJloCqpD9x99MlnS3pnWZ2uqT3SrrJ3ddIuimsS9KFktaE12WSPi3loUHS5ZKeL+ksSZc3gkO3KJcSBgYCALrWtIcAd9/m7neF5X2S7pO0XNJFkq4Ou10t6TVh+SJJX/DcrZLmm9lSSS+TdKO773L33ZJulHTB9J3J0ZXTRBVaAgAAXaqjYwLMbJWkZ0u6TdISd98WNm2XtCQsL5e0ueljW0LZROVdI6MlAADQxToWAsxstqSvSXqPu+9t3ubuLqmwEXVmdpmZrTOzdYODg0Ud9qjygYGEAABAd+pICDCzTHkA+LK7fz0U7wjN/ArvO0P5Vkkrmz6+IpRNVP4k7n6lu69197UDAwPFnchRlNNEI1VmBwAAulMnZgeYpM9Kus/dP9a06XpJjRH+l0j6ZlP5xWGWwNmSngjdBjdIeqmZLQgDAl8ayrpGVqIlAADQvUod+M4XSPptSfea2U9C2V9I+pCka83sUkkPS3pj2PYdSS+XtEnSQUnvkCR332VmfyvpjrDf+91917ScwTEqM0UQANDFpj0EuPuPJNkEm88fZ3+X9M4JjnWVpKuKq12xyqVEwxVCAACgO3HHwDZiYCAAoJsRAtooSxMN0x0AAOhShIA2KjMwEADQxQgBbVTmAUIAgC5GCGijcpro4EhV+dhGAAC6CyGgjZ57ygI9tn9Etz7QVTMXAQCQRAhoq1efuUzzZ2a6+paHOl0VAACehBDQRv1Zqjc9b6W+t2G7tu451OnqAABwBEJAm/322adIkr5068MdrgkAAEciBLTZigUz9eJfXqJrbn9EQ5Vap6sDAMAoQsA0ePu5q7T7YEXX//TRTlcFAIBRhIBpcM4vnaSnLpmtq295iOmCAICuQQiYBmami89ZpfWP7tWdD+/udHUAAJBECJg2r332cs3pL+nzTBcEAHQJQsA0mdVX0hvXrtR3f7ZdO/YOdbo6AAAQAqbTxeecopq7vsx0QQBAFyAETKNTTpqlFz1tsb5y+yMarjJdEADQWYSAaXbJuav02P4RfefebZ2uCgCgxxECptmvrV6k0wZm6fO30CUAAOgsQsA0SxLTJees0k8379FPNu/pdHUAAD2MENABr3/uCs3uK/F0QQBARxECOmB2X0lveO4KfeueRzW4b7jT1QEA9ChCQIdcfM4pqtRcX739kU5XBQDQowgBHXLawGz9+lMH9KVbH9ZItd7p6gAAehAhoIN+99dO1c59w/qrf72XBwsBAKYdIaCDfm3NgN513mpdu26LPv0fv+h0dQAAPabU6Qr0uj96yVP10OMH9ZHv3q+nLJypV56xrNNVAgD0CFoCOszM9JE3nKG1pyzQH1/7Ux41DACYNoSALtCfpbry4rVaOq9fl31hnR55/GCnqwQA6AGEgC6xcFZZn3v781Stu97x+dv1xMFKp6sEADjBEQK6yGkDs3Xlbz9Xj+w6qN/70p1MHQQAtBUhoMs8/7ST9OHXn6EfP/C4/vIbTB0EALQPswO60Oues0IPP35Qn7xpo1YtmqV3vmh1p6sEADgBEQK61HtevEaP7Dqov7vhfplJl/3aaSqlNNwAAIrDX5UuZWb60Ot/RRc+82R95Lv363WfvkX3bdvb6WoBAE4ghIAu1ldK9Y9vfY7+/reera27D+lVn/qRPnbjf2u4Wut01QAAJwBCQJczM73yjGX69z/+Db3qWct0xU0b9apP/Uh3P8JNhQAArSEERGLBrLI+/qYz9bm3P0/7hqp6/adv0Qe+tUGHRmgVAABMDSEgMi96+mJ9749+Xb/1/KfoMz96UC/7xA91w/rtqta4pwAA4PhYr81DX7t2ra9bt67T1SjErQ88rvd+7R499PhBLZrdp9ecuUxvWLtCTz95bqerBgDoEmZ2p7uvHXcbISBulVpdP7h/UNfduVnf//lOVWquZyybqzc8d4UuOnO5Fs4qd7qKAIAOIgQ0OdFCQLNdB0Z0/U+26rq7tuhnW/eqlJjOe/pi/ebalTrv6YuVJtbpKgIAphkhoMmJHAKa/Xz7Xn3tzi36xt2P6rH9w1o+f4YuPucUvel5KzV/Jq0DANArCAFNeiUENFRrdf37fTv0uf96SLc9uEv9WaLXPnu5Ljl3FWMHAKAHEAKa9FoIaHbftr26+paH9I27t2q4WtfZpy3U289dpRf/8hJuSQwAJyhCQJNeDgENuw+M6J/XbdYXf/ywtu45pOXzZ+iiM5fpFWcs1elL58qMsQMAcKIgBDQhBByWdxXs1Jdve1i3/OJx1equ0xbN0ivOWKpXnLFUT1syh0AAAJEjBDQhBIzv8f3DumH9Dn373kf14188rrpLqxfP1it+Zale9aylWr14TqerCACYAkJAE0LA0Q3uG9Z312/Xt+95VLc9uEvu0qLZZa1ePDt/DczWmiVztHrxbC2e00drAQB0MUJAE0LA8dm5d0g3rN+ue7c+oU0792vjzv3aN1Qd3T6nv6TVi2fraUvm6PRlc/WMZXP19JPnalZfqYO1BgA0EAKaEAJa4+4a3DesjTv3a1N4bdy5T/dv36fdByuSJDPp1EWz9Ixl8/SMZXN1+tI8HJw0u6/DtQeA3jNZCODnGo6LmWnx3H4tntuvF6xeNFru7tr2xJDWP7pX6x99Qusf3au7Ht6tf/vpo6P7nDSrrDVLZuupS+ZozeK8S+GpS+Zwa2MA6BBCAAphZlo2f4aWzZ+hl5y+ZLR8z8ERbXh0rzZs26uNO/brv3fu09fv2qr9w4e7FBrhYMWCmVo6r19L583Q0nn9Onlev5bO69e8GRnjDgCgDQgBaKv5M8s6d/UinTtOq8HGnfu1ccc+/feOfdq4c79+tPEx7dw3pPqYHqr+LNHSeTO0bH6/Vi6YqZULZ2rFghlauXCmVi6YqUWzy4QEAJgCQgCmXXOrwW88deCIbdVaXYP7h/XoniFtf2JI2544FN6HtGXPId24YYcePzByxGdmZKlWLsyPt3BmWQtmlbVwVlkLZpa1YGZ2xPrCWWUepAQAASEAXaWUJqE7YMaE+xwYrmrL7kPavOugNu8+qM27Dmnz7oPa9sQhbdyxX7sPjujgSG3cz6aJaWB2n5bM7dPiuf1aMrdPS+b0a8ncfi2e26eTZvVp/sxM82dmmt1XooUBwAmNEIDozOor6Wknz9HTTp74BkZDlZr2HKxo14ER7T44ol0H8tfgvmHt2DukHfuGtXnXQa17aNforIaxSomFQFDW/Bn5+6y+VKmZksSUWB4qErPR975SokWz+7R4bp8WzwkhY24/UyYBdCX+y4QTUn+W6uR5qU6e13/UfYertRAOhrXrwIj2HBzRnoMV7T44oj2HKtpzcES7D1S0dc8hHRypqlZ3uUu1uqvmLnfPl+uu4Wpdw9X6k75jVjnVkrn9GpjTpwUzy5o3I9O8mZnm9pc0b0amueE1b0beAjEjS9WfpZpRTtVfSnjAE4C2IASg5/WVUq1YMFMrFsxs+Vjurn3DVe3cO6yde4e0Y9+QduzNWx927svLfjG4X3uHKnriUEVDlScHhvFkqeWhIEs1s5xqTn+muTNKmtMX3vszze3PNKe/pDn9JfVlqcppor4sUV+aqFzKX32lVOVSopnlVLP6SpqZpUoYIwH0LEIAUCAz09zwB3n14tlH3X+4WtPeQ1U9cSgPBXsPVXRwpKZDlfw11LR8aKSmoUpNB0Zq2jeU77tz77D2DVW1d6gy4TiIo2kEgtl9Jc3qSzWzXNLMcqq+UqJyqfGeNL3nZX2lRH1Z3lIx3nvzPkccI00YawF0CUIA0EF9pVQDc1INzGn9boqVWl37h6raP1zVcLWmoUpdI7W6Rqr5a3j0PQ8VB4ar2j9c08Hhqg6MHF7eP1zVrgMjR3xuuFob7eoYGae74/jPO7ROpImy0FKRpaYszYNCdkTrRVPwyA4vl8N+aWIqJfm4jFKaHF5OTOVSov5S3rXSl+XLze/lNFEpNZWSxrsRUNBTCAHACSJLEy2YlU+RbCf3fOzDSK2uoUpNw5U8HAxVaocDQ6UpOFSODBF5WW00oFRqdVVqnoeOWmM9/9z+4eoRn28OJZVae2553ggQpRAqsqaQkIWQcbj8yH3G2zcLYac56DSHncZ+SXOYGQ01pjRJmupz5PHTxJQlidLUwoBVKbV8nyQ5PHC1cTwCDsYiBAA4Lmb5+IT+LNXc/qxj9ajXXdUwILNar6tae/L6aFAJIWUoBIrG+0j18OeqtXr+Xg/vtbys0thW89HlSi3frxLKhyp1VWvV0fJ83/A+Gnby5erYu2FNo7EhJQ8ReZBJLP/f1kxKLJ/9koTQkJiNtpSUkmQ0oOThIg83adJ0zLQRZhrbQtAxSU3HNklJcvg7m0NQaWwrT9KooyTln8kPZ+Fdo0GquWXn8Pnm9Rj97qbZPYfPeUwI64HwRAgAEKUkMZVHBzWmHa3L8ajVfbS1oxEOau6qNd6bQkhzoKnV8xBSq4cQ0hQ4ak0zVBqvuh8ORbX6kYGm0nTsSgg7LqnueUuPu1R3D6+8bDRghe8dqoY6hc/X6keGqcZ5Hq53PpOmgxloyhLTaPhJm0JLGoKNWWiJCctSHkryoGJqZAjT4c+NbelpDh1vO/sUvewZJ0/LuUUfAszsAkmfVP5fgc+4+4c6XCUAmFD+ByBvSelV3hQu6k2Bo1Y/HCwaIaV5vR4Ciit/z491eL0RfJoD0thWnny/vCWp3vT9o1N9XaNBrNbUulTz/Dij9a0fWfd6XYdDk1zh/9R4Uq+Hujb2PxzymgNZXYcqXsi4m2MVdQgws1TSP0h6iaQtku4ws+vdfUNnawYAmEj+y1nKfxujk2K/A8lZkja5+wPuPiLpGkkXdbhOAABEIfYQsFzS5qb1LaHsCGZ2mZmtM7N1g4OD01Y5AAC6Wewh4Ji4+5Xuvtbd1w4MDBz9AwAA9IDYQ8BWSSub1leEMgAAcBSxh4A7JK0xs1PNrCzpzZKu73CdAACIQtSzA9y9amZ/IOkG5VMEr3L39R2uFgAAUYg6BEiSu39H0nc6XQ8AAGITe3cAAACYIkIAAAA9ihAAAECPIgQAANCjCAEAAPQoQgAAAD2KEAAAQI8iBAAA0KMIAQAA9ChCAAAAPYoQAABAjzJ373QdppWZDUp6uMBDLpL0WIHH62Vcy+JwLYvBdSwO17I4x3stT3H3gfE29FwIKJqZrXP3tZ2ux4mAa1kcrmUxuI7F4VoWp8hrSXcAAAA9ihAAAECPIgS07spOV+AEwrUsDteyGFzH4nAti1PYtWRMAAAAPYqWAAAAehQhYIrM7AIzu9/MNpnZeztdn5iY2VVmttPMftZUttDMbjSzjeF9QSfrGAszW2lmN5vZBjNbb2bvDuVcz+NkZv1mdruZ/TRcy78J5aea2W3h3/o/m1m503WNgZmlZna3mX0rrHMdp8DMHjKze83sJ2a2LpQV9u+bEDAFZpZK+gdJF0o6XdJbzOz0ztYqKp+XdMGYsvdKusnd10i6Kazj6KqS/sTdT5d0tqR3hv9f5Hoev2FJ57n7sySdKekCMztb0oclfdzdV0vaLenSzlUxKu+WdF/TOtdx6l7k7mc2TQss7N83IWBqzpK0yd0fcPcRSddIuqjDdYqGu/9Q0q4xxRdJujosXy3pNdNZp1i5+zZ3vyss71P+H93l4noeN8/tD6tZeLmk8yRdF8q5lsfAzFZIeoWkz4R1E9exSIX9+yYETM1ySZub1reEMkzdEnffFpa3S1rSycrEyMxWSXq2pNvE9ZyS0IT9E0k7Jd0o6ReS9rh7NezCv/Vj8wlJ/0tSPayfJK7jVLmk75nZnWZ2WSgr7N93qdXaAUVzdzczpq0cBzObLelrkt7j7nvzH145ruexc/eapDPNbL6kb0h6emdrFB8ze6Wkne5+p5m9sMPVORH8qrtvNbPFkm40s583b2z13zctAVOzVdLKpvUVoQxTt8PMlkpSeN/Z4fpEw8wy5QHgy+7+9VDM9WyBu++RdLOkcyTNN7PGDyb+rR/dCyS92sweUt5Vep6kT4rrOCXuvjW871QeTM9Sgf++CQFTc4ekNWG0a1nSmyVd3+E6xe56SZeE5UskfbODdYlG6Gv9rKT73P1jTZu4nsfJzAZCC4DMbIaklygfY3GzpDeE3biWR+Hu73P3Fe6+Svl/G7/v7m8V1/G4mdksM5vTWJb0Ukk/U4H/vrlZ0BSZ2cuV93ulkq5y9w92tkbxMLOvSnqh8idh7ZB0uaR/lXStpKcof8rjG9197OBBjGFmvyrpPyXdq8P9r3+hfFwA1/M4mNkZygdZpcp/IF3r7u83s9OU/6JdKOluSW9z9+HO1TQeoTvgT939lVzH4xeu2TfCaknSV9z9g2Z2kgr6900IAACgR9EdAABAjyIEAADQowgBAAD0KEIAAAA9ihAAAECPIgQAOCozq4WnmDVehT2QyMxWNT9REsD04bbBAI7FIXc/s9OVAFAsWgIATFl41vlHwvPObzez1aF8lZl938zuMbObzOwpoXyJmX3DzH4aXueGQ6Vm9k9mtt7Mvhfu2Ccze5eZbQjHuaZDpwmcsAgBAI7FjDHdAW9q2vaEu/+KpL9XfhdNSfqUpKvd/QxJX5Z0RSi/QtJ/uPuzJD1H0vpQvkbSP7j7MyTtkfT6UP5eSc8Ox/m99pwa0Lu4YyCAozKz/e4+e5zyhySd5+4PhAcZbXf3k8zsMUlL3b0Syre5+yIzG5S0ovl2seERyDe6+5qw/ueSMnf/gJl9V9J+5beV/ld339/mUwV6Ci0BAFrlEywfj+Z7yNd0eLzSKyT9g/JWgzuankIHoACEAACtelPT+4/D8i3KnyAnSW9V/pAjSbpJ0u9LkpmlZjZvooOaWSJppbvfLOnPJc2T9KTWCABTR6oGcCxmmNlPmta/6+6NaYILzOwe5b/m3xLK/lDS58zszyQNSnpHKH+3pCvN7FLlv/h/X9K2Cb4zlfSlEBRM0hXuvqeg8wEgxgQAaEEYE7DW3R/rdF0AHD+6AwAA6FG0BAAA0KNoCQAAoEcRAgAA6FGEAAAAehQhAACAHkUIAACgRxECAADoUf8fZMWdL4vnXhwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "layers = [isize, 32, osize]\n",
    "network = Network(layers, activation_choice='sigmoid',\n",
    "                output_choice='softmax', loss_choice='cce')\n",
    "\n",
    "epochs = 50\n",
    "network.fit(X_train, Y_train, lr=0.01, epochs=epochs, batch_size=10)\n",
    "\n",
    "#plot losses\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.plot(range(epochs), network.losses)\n",
    "plt.title('Loss vs Epochs')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('CCE Loss')\n",
    "\n",
    "#compute accuracy\n",
    "accuracy = np.sum(network.predict(X_test) == y_test )/ X_test.shape[0]\n",
    "print(f'Test-data size = {X_test.shape[0]}')\n",
    "print(f'Accuracy = {accuracy: .2f}')\n",
    "print(f'Number of parameters = {count_params(layers)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "369f2c481f4da34e4445cda3fffd2e751bd1c4d706f27375911949ba6bb62e1c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}