notebooks/03_feedforward_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Users\\Yue\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.datasets import mnist\n",
    "from tensorflow.contrib.eager.python import tfe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# enable eager mode\n",
    "tf.enable_eager_execution()\n",
    "tf.set_random_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants\n",
    "hidden_dim = 500\n",
    "batch_size = 128\n",
    "epochs = 10\n",
    "num_classes = 10\n",
    "\n",
    "if not os.path.exists('weights/'):\n",
    "    os.makedirs('weights/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x train (60000, 784)\n",
      "y train (60000, 10)\n",
      "x test (10000, 784)\n",
      "y test (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# dataset loading\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "# normalization of dataset\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "\n",
    "# flatten the dataset\n",
    "x_train = x_train.reshape((-1, 28 * 28))\n",
    "x_test = x_test.reshape((-1, 28 * 28))\n",
    "\n",
    "# one hot encode the labels. convert back to numpy as we cannot use a combination of numpy\n",
    "# and tensors as input to keras\n",
    "y_train_ohe = tf.one_hot(y_train, depth=num_classes).numpy()\n",
    "y_test_ohe = tf.one_hot(y_test, depth=num_classes).numpy()\n",
    "\n",
    "print('x train', x_train.shape)\n",
    "print('y train', y_train_ohe.shape)\n",
    "print('x test', x_test.shape)\n",
    "print('y test', y_test_ohe.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi Layered Perceptron\n",
    "\n",
    "Extremely simple to build, yet powerful enough for MNIST. Easily manages 95% test accuracy in less than 5 epochs. Then again, MNIST is not nearly challenging enough.\n",
    "\n",
    "Something to note : We can chain activation for all the intermediate layers when creating the layers. Dont bother with `tf.keras.layers.Activation()` unless you are doing a `Conv-BatchNorm-Relu` block, which will be shown later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model definition (canonical way)\n",
    "class MLP(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, hidden_units, num_classes):\n",
    "        super(MLP, self).__init__()\n",
    "\n",
    "        self.hidden = tf.keras.layers.Dense(hidden_units, activation='relu')\n",
    "        self.classifier = tf.keras.layers.Dense(num_classes)\n",
    "\n",
    "    def call(self, inputs, training=None, mask=None):\n",
    "        x = self.hidden(inputs)\n",
    "        output = self.classifier(x)\n",
    "\n",
    "        # softmax op does not exist on the gpu, so always use cpu\n",
    "        with tf.device('/cpu:0'):\n",
    "            output = tf.nn.softmax(output)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      " - 6s - loss: 0.4373 - acc: 0.8836 - val_loss: 0.2715 - val_acc: 0.9245\n",
      "Epoch 2/10\n",
      " - 6s - loss: 0.2462 - acc: 0.9314 - val_loss: 0.2105 - val_acc: 0.9406\n",
      "Epoch 3/10\n",
      " - 6s - loss: 0.1967 - acc: 0.9451 - val_loss: 0.1797 - val_acc: 0.9485\n",
      "Epoch 4/10\n",
      " - 6s - loss: 0.1647 - acc: 0.9538 - val_loss: 0.1510 - val_acc: 0.9564\n",
      "Epoch 5/10\n",
      " - 6s - loss: 0.1414 - acc: 0.9604 - val_loss: 0.1390 - val_acc: 0.9607\n",
      "Epoch 6/10\n",
      " - 6s - loss: 0.1235 - acc: 0.9661 - val_loss: 0.1234 - val_acc: 0.9653\n",
      "Epoch 7/10\n",
      " - 5s - loss: 0.1098 - acc: 0.9696 - val_loss: 0.1128 - val_acc: 0.9675\n",
      "Epoch 8/10\n",
      " - 5s - loss: 0.0984 - acc: 0.9729 - val_loss: 0.1051 - val_acc: 0.9691\n",
      "Epoch 9/10\n",
      " - 5s - loss: 0.0890 - acc: 0.9759 - val_loss: 0.1000 - val_acc: 0.9722\n",
      "Epoch 10/10\n",
      " - 6s - loss: 0.0814 - acc: 0.9778 - val_loss: 0.0920 - val_acc: 0.9740\n",
      "Final test loss and accuracy : [0.09196617434620857, 0.974]\n"
     ]
    }
   ],
   "source": [
    "device = '/cpu:0' if tfe.num_gpus() == 0 else '/gpu:0'\n",
    "\n",
    "with tf.device(device):\n",
    "    # build model and optimizer\n",
    "    model = MLP(hidden_dim, num_classes)\n",
    "    model.compile(optimizer=tf.train.GradientDescentOptimizer(0.1), loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    # TF Keras tries to use entire dataset to determine shape without this step when using .fit()\n",
    "    # Fix = Use exactly one sample from the provided input dataset to determine input/output shape/s for the model\n",
    "    dummy_x = tf.zeros((1, 28 * 28))\n",
    "    model._set_inputs(dummy_x)\n",
    "\n",
    "    # train\n",
    "    model.fit(x_train, y_train_ohe, batch_size=batch_size, epochs=epochs,\n",
    "              validation_data=(x_test, y_test_ohe), verbose=2)\n",
    "\n",
    "    # evaluate on test set\n",
    "    scores = model.evaluate(x_test, y_test_ohe, batch_size, verbose=2)\n",
    "    print(\"Final test loss and accuracy :\", scores)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving weights\n",
    "Saving weights for Eager models which are built solely with Keras Layers or Keras Models is super simple.\n",
    "\n",
    "I havent tried mixing tf.layers API with Eager tf.keras. Something to try once the API matures.\n",
    "\n",
    "As easy as it is to save models, there are several pain points with restoring models though, which are discussed in the final tutorial - `10_custom_models.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'weights/03_feedforward/weights.ckpt'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the weights of the model. This is same for all models defined after this.\n",
    "saver = tfe.Saver(model.variables)\n",
    "saver.save('weights/03_feedforward/weights.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}