U2.15 - [TALLER] - Weights initialization - SOLUTION.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget --no-cache -O init.py -q https://raw.githubusercontent.com/rramosp/2020.deeplearning/master/init.py\n",
    "from init import init; init(force_download=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.1.0'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "if 'google.colab' in sys.modules:\n",
    "    print (\"setting tensorflow version in colab\")\n",
    "    %tensorflow_version 2.x\n",
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LAB Weights initizialization\n",
    "\n",
    "perform the following experiments following the code in the corresponding Weights Initialization notebook:\n",
    "\n",
    "- with $\\sigma$=0.001\n",
    "- with $\\sigma$=100\n",
    "\n",
    "using **no scaling** of input data and keeping the rest of parameters (network structure, epochs, etc.).\n",
    "\n",
    "\n",
    "**<font color=\"red\">USE TENSORBOARD</font>** to record your experiments and **<font color=\"red\">include in this notebook screenshots</font>** from tensorboard's visualization of your experiments. In particular include, for each experiment:\n",
    "\n",
    "- the **scalar** chart of train and test accuracy\n",
    "- the **histograms** of weights, biases and their respective gradients of layer_0\n",
    "- the **distributions** of weights, biases and their respective gradients of layer_0\n",
    "\n",
    "Then **<font color=\"red\">ADD YOUR INTERPRETATION EXPLAINING THE OBSERVED BEHAVIOURS</font>**\n",
    "\n",
    "The charts should look roughly like this\n",
    "\n",
    "![alt text](local/imgs/lab_winit_01.png)\n",
    "![alt text](local/imgs/lab_winit_02.png)\n",
    "![alt text](local/imgs/lab_winit_03.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from tensorflow import keras\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dimension de las imagenes y las clases (1500, 784) (1500,)\n",
      "(1200, 784) (1200, 10)\n"
     ]
    }
   ],
   "source": [
    "mnist = pd.read_csv(\"local/data/mnist1.5k.csv.gz\", compression=\"gzip\", header=None).values\n",
    "X=mnist[:,1:785]/255.\n",
    "y=mnist[:,0]\n",
    "print (\"dimension de las imagenes y las clases\", X.shape, y.shape)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)\n",
    "X_train = X_train\n",
    "X_test  = X_test\n",
    "y_train_oh = np.eye(10)[y_train]\n",
    "y_test_oh  = np.eye(10)[y_test]\n",
    "print (X_train.shape, y_train_oh.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.1.0'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Flatten, concatenate, Input\n",
    "from tensorflow.keras.backend import clear_session\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(input_dim=784, output_dim=10, layer_sizes=[10]*6, activation=\"relu\", sigma=1):\n",
    "    \n",
    "    \n",
    "    clear_session()\n",
    "    model = Sequential()\n",
    "    init1k = keras.initializers.RandomNormal(mean=.0, stddev=sigma, seed=None)\n",
    "    init1b = keras.initializers.RandomNormal(mean=.0, stddev=sigma, seed=None)\n",
    "\n",
    "    model.add(Dense(layer_sizes[0], activation=activation, input_dim=input_dim, name=\"Layer_%02d_Input\"%(0),\n",
    "                    kernel_initializer=init1k,\n",
    "                    bias_initializer=init1b\n",
    "                ))\n",
    "\n",
    "   \n",
    "    for i, hsize in enumerate(layer_sizes[1:]):\n",
    "        model.add(Dense(hsize, activation=activation, name=\"Layer_%02d_Hidden\"%(i+1)))\n",
    "   \n",
    "    model.add(Dense(output_dim, activation=\"softmax\", name=\"Layer_%02d_Output\"%(len(layer_sizes))))\n",
    "        \n",
    "    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    model.reset_states()\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_model(layer_sizes=[20,15,15], activation=\"sigmoid\", sigma=20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0220 18:17:27.399759 4600747456 callbacks.py:1501] `write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1200 samples, validate on 300 samples\n",
      "Epoch 1/30\n",
      "1200/1200 [==============================] - 0s 394us/sample - loss: 2.3454 - accuracy: 0.1100 - val_loss: 2.3153 - val_accuracy: 0.0833\n",
      "Epoch 2/30\n",
      "1200/1200 [==============================] - 0s 65us/sample - loss: 2.2833 - accuracy: 0.1417 - val_loss: 2.2767 - val_accuracy: 0.1567\n",
      "Epoch 3/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 2.2511 - accuracy: 0.2083 - val_loss: 2.2520 - val_accuracy: 0.1933\n",
      "Epoch 4/30\n",
      "1200/1200 [==============================] - 0s 69us/sample - loss: 2.2246 - accuracy: 0.2283 - val_loss: 2.2264 - val_accuracy: 0.2467\n",
      "Epoch 5/30\n",
      "1200/1200 [==============================] - 0s 65us/sample - loss: 2.1931 - accuracy: 0.2758 - val_loss: 2.1958 - val_accuracy: 0.2467\n",
      "Epoch 6/30\n",
      "1200/1200 [==============================] - 0s 61us/sample - loss: 2.1558 - accuracy: 0.2700 - val_loss: 2.1596 - val_accuracy: 0.2733\n",
      "Epoch 7/30\n",
      "1200/1200 [==============================] - 0s 67us/sample - loss: 2.1099 - accuracy: 0.3367 - val_loss: 2.1162 - val_accuracy: 0.3133\n",
      "Epoch 8/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 2.0589 - accuracy: 0.3583 - val_loss: 2.0732 - val_accuracy: 0.3533\n",
      "Epoch 9/30\n",
      "1200/1200 [==============================] - 0s 63us/sample - loss: 2.0048 - accuracy: 0.3892 - val_loss: 2.0276 - val_accuracy: 0.3667\n",
      "Epoch 10/30\n",
      "1200/1200 [==============================] - 0s 62us/sample - loss: 1.9502 - accuracy: 0.3808 - val_loss: 1.9847 - val_accuracy: 0.3567\n",
      "Epoch 11/30\n",
      "1200/1200 [==============================] - 0s 78us/sample - loss: 1.8960 - accuracy: 0.4383 - val_loss: 1.9404 - val_accuracy: 0.4033\n",
      "Epoch 12/30\n",
      "1200/1200 [==============================] - 0s 67us/sample - loss: 1.8453 - accuracy: 0.4425 - val_loss: 1.9010 - val_accuracy: 0.4000\n",
      "Epoch 13/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 1.7918 - accuracy: 0.4983 - val_loss: 1.8604 - val_accuracy: 0.4500\n",
      "Epoch 14/30\n",
      "1200/1200 [==============================] - 0s 68us/sample - loss: 1.7406 - accuracy: 0.5133 - val_loss: 1.8119 - val_accuracy: 0.4567\n",
      "Epoch 15/30\n",
      "1200/1200 [==============================] - 0s 75us/sample - loss: 1.6860 - accuracy: 0.5342 - val_loss: 1.7642 - val_accuracy: 0.4600\n",
      "Epoch 16/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 1.6319 - accuracy: 0.5608 - val_loss: 1.7182 - val_accuracy: 0.4967\n",
      "Epoch 17/30\n",
      "1200/1200 [==============================] - 0s 61us/sample - loss: 1.5764 - accuracy: 0.5633 - val_loss: 1.6744 - val_accuracy: 0.4967\n",
      "Epoch 18/30\n",
      "1200/1200 [==============================] - 0s 65us/sample - loss: 1.5219 - accuracy: 0.5958 - val_loss: 1.6282 - val_accuracy: 0.5267\n",
      "Epoch 19/30\n",
      "1200/1200 [==============================] - 0s 54us/sample - loss: 1.4690 - accuracy: 0.6217 - val_loss: 1.5840 - val_accuracy: 0.5467\n",
      "Epoch 20/30\n",
      "1200/1200 [==============================] - 0s 59us/sample - loss: 1.4183 - accuracy: 0.6367 - val_loss: 1.5397 - val_accuracy: 0.5600\n",
      "Epoch 21/30\n",
      "1200/1200 [==============================] - 0s 52us/sample - loss: 1.3671 - accuracy: 0.6558 - val_loss: 1.4972 - val_accuracy: 0.5633\n",
      "Epoch 22/30\n",
      "1200/1200 [==============================] - 0s 50us/sample - loss: 1.3198 - accuracy: 0.6683 - val_loss: 1.4517 - val_accuracy: 0.5733\n",
      "Epoch 23/30\n",
      "1200/1200 [==============================] - 0s 47us/sample - loss: 1.2693 - accuracy: 0.6958 - val_loss: 1.4164 - val_accuracy: 0.5867\n",
      "Epoch 24/30\n",
      "1200/1200 [==============================] - 0s 51us/sample - loss: 1.2245 - accuracy: 0.7083 - val_loss: 1.3710 - val_accuracy: 0.6033\n",
      "Epoch 25/30\n",
      "1200/1200 [==============================] - 0s 72us/sample - loss: 1.1805 - accuracy: 0.7275 - val_loss: 1.3332 - val_accuracy: 0.6300\n",
      "Epoch 26/30\n",
      "1200/1200 [==============================] - 0s 73us/sample - loss: 1.1385 - accuracy: 0.7400 - val_loss: 1.2961 - val_accuracy: 0.6400\n",
      "Epoch 27/30\n",
      "1200/1200 [==============================] - 0s 68us/sample - loss: 1.0947 - accuracy: 0.7600 - val_loss: 1.2677 - val_accuracy: 0.6467\n",
      "Epoch 28/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 1.0563 - accuracy: 0.7808 - val_loss: 1.2276 - val_accuracy: 0.6667\n",
      "Epoch 29/30\n",
      "1200/1200 [==============================] - 0s 64us/sample - loss: 1.0158 - accuracy: 0.7875 - val_loss: 1.1999 - val_accuracy: 0.6733\n",
      "Epoch 30/30\n",
      "1200/1200 [==============================] - 0s 56us/sample - loss: 0.9791 - accuracy: 0.7967 - val_loss: 1.1577 - val_accuracy: 0.7067\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x149d40e48>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!rm -rf log\n",
    "model = get_model(layer_sizes=[20,15,15], activation=\"sigmoid\", sigma=.001)\n",
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./log/sigma_0.001', histogram_freq=1,  write_grads=True, write_graph=True)\n",
    "model.fit(X_train, y_train_oh, epochs=30, batch_size=32, validation_data=(X_test, y_test_oh), callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1200 samples, validate on 300 samples\n",
      "Epoch 1/30\n",
      "1200/1200 [==============================] - 0s 278us/step - loss: 2.3334 - acc: 0.1308 - val_loss: 2.3279 - val_acc: 0.0967\n",
      "Epoch 2/30\n",
      "1200/1200 [==============================] - 0s 35us/step - loss: 2.3025 - acc: 0.1308 - val_loss: 2.3114 - val_acc: 0.0967\n",
      "Epoch 3/30\n",
      "1200/1200 [==============================] - 0s 35us/step - loss: 2.2897 - acc: 0.1308 - val_loss: 2.3038 - val_acc: 0.0967\n",
      "Epoch 4/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.2832 - acc: 0.1308 - val_loss: 2.3014 - val_acc: 0.0967\n",
      "Epoch 5/30\n",
      "1200/1200 [==============================] - 0s 35us/step - loss: 2.2771 - acc: 0.1308 - val_loss: 2.2983 - val_acc: 0.0967\n",
      "Epoch 6/30\n",
      "1200/1200 [==============================] - 0s 35us/step - loss: 2.2704 - acc: 0.1333 - val_loss: 2.2904 - val_acc: 0.0967\n",
      "Epoch 7/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.2637 - acc: 0.1317 - val_loss: 2.2863 - val_acc: 0.0967\n",
      "Epoch 8/30\n",
      "1200/1200 [==============================] - 0s 31us/step - loss: 2.2567 - acc: 0.1575 - val_loss: 2.2789 - val_acc: 0.1267\n",
      "Epoch 9/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.2478 - acc: 0.1833 - val_loss: 2.2707 - val_acc: 0.1367\n",
      "Epoch 10/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.2377 - acc: 0.2125 - val_loss: 2.2631 - val_acc: 0.1500\n",
      "Epoch 11/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.2271 - acc: 0.2242 - val_loss: 2.2514 - val_acc: 0.1533\n",
      "Epoch 12/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.2147 - acc: 0.2517 - val_loss: 2.2416 - val_acc: 0.1600\n",
      "Epoch 13/30\n",
      "1200/1200 [==============================] - 0s 34us/step - loss: 2.2018 - acc: 0.2475 - val_loss: 2.2302 - val_acc: 0.1533\n",
      "Epoch 14/30\n",
      "1200/1200 [==============================] - 0s 31us/step - loss: 2.1876 - acc: 0.2508 - val_loss: 2.2150 - val_acc: 0.1833\n",
      "Epoch 15/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.1733 - acc: 0.2575 - val_loss: 2.2048 - val_acc: 0.1867\n",
      "Epoch 16/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.1582 - acc: 0.2558 - val_loss: 2.1918 - val_acc: 0.1933\n",
      "Epoch 17/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.1421 - acc: 0.2658 - val_loss: 2.1773 - val_acc: 0.2000\n",
      "Epoch 18/30\n",
      "1200/1200 [==============================] - 0s 34us/step - loss: 2.1263 - acc: 0.2667 - val_loss: 2.1623 - val_acc: 0.2000\n",
      "Epoch 19/30\n",
      "1200/1200 [==============================] - 0s 32us/step - loss: 2.1108 - acc: 0.2625 - val_loss: 2.1495 - val_acc: 0.2033\n",
      "Epoch 20/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.0954 - acc: 0.2892 - val_loss: 2.1343 - val_acc: 0.2133\n",
      "Epoch 21/30\n",
      "1200/1200 [==============================] - 0s 34us/step - loss: 2.0799 - acc: 0.2900 - val_loss: 2.1215 - val_acc: 0.2233\n",
      "Epoch 22/30\n",
      "1200/1200 [==============================] - 0s 42us/step - loss: 2.0646 - acc: 0.2975 - val_loss: 2.1057 - val_acc: 0.2300\n",
      "Epoch 23/30\n",
      "1200/1200 [==============================] - 0s 39us/step - loss: 2.0493 - acc: 0.2992 - val_loss: 2.0918 - val_acc: 0.2300\n",
      "Epoch 24/30\n",
      "1200/1200 [==============================] - 0s 33us/step - loss: 2.0352 - acc: 0.2967 - val_loss: 2.0818 - val_acc: 0.2300\n",
      "Epoch 25/30\n",
      "1200/1200 [==============================] - 0s 73us/step - loss: 2.0218 - acc: 0.3133 - val_loss: 2.0649 - val_acc: 0.2400\n",
      "Epoch 26/30\n",
      "1200/1200 [==============================] - 0s 68us/step - loss: 2.0055 - acc: 0.3192 - val_loss: 2.0520 - val_acc: 0.2400\n",
      "Epoch 27/30\n",
      "1200/1200 [==============================] - 0s 84us/step - loss: 1.9915 - acc: 0.3158 - val_loss: 2.0411 - val_acc: 0.2400\n",
      "Epoch 28/30\n",
      "1200/1200 [==============================] - 0s 71us/step - loss: 1.9767 - acc: 0.3275 - val_loss: 2.0252 - val_acc: 0.2433\n",
      "Epoch 29/30\n",
      "1200/1200 [==============================] - 0s 76us/step - loss: 1.9629 - acc: 0.3300 - val_loss: 2.0117 - val_acc: 0.2567\n",
      "Epoch 30/30\n",
      "1200/1200 [==============================] - 0s 46us/step - loss: 1.9485 - acc: 0.3333 - val_loss: 1.9979 - val_acc: 0.2633\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f7631944910>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_model(layer_sizes=[20,15,15], activation=\"sigmoid\", sigma=100)\n",
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./log/sigma_100', histogram_freq=1,  write_grads=True, write_graph=True)\n",
    "model.fit(X_train, y_train_oh, epochs=30, batch_size=32, validation_data=(X_test, y_test_oh), callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "p37",
   "language": "python",
   "name": "p37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}