train/MNIST_fully_connected_feed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Builds the MNIST network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import math\n",
    "import argparse\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from six.moves import xrange\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The MNIST dataset has 10 classes, representing the digits 0 through 9.\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# The MNIST images are always 28x28 pixels.\n",
    "IMAGE_SIZE = 28\n",
    "IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE\n",
    "\n",
    "batch_size = 100\n",
    "fake_data = False\n",
    "hidden1 = 128\n",
    "hidden2 = 32\n",
    "max_steps = 5000\n",
    "learning_rate = 0.01\n",
    "log_dir = os.path.join('.', 'logs')\n",
    "quantize_delay = 4500\n",
    "input_data_dir = os.path.join('.', 'input_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_inference(images, hidden1_units, hidden2_units):\n",
    "    \"\"\"Build the MNIST model up to where it may be used for inference.\n",
    "    Args:\n",
    "        images: Images placeholder, from inputs().\n",
    "        hidden1_units: Size of the first hidden layer.\n",
    "        hidden2_units: Size of the second hidden layer.\n",
    "    Returns:\n",
    "        softmax_linear: Output tensor with the computed logits.\n",
    "    \"\"\"\n",
    "    # Hidden 1\n",
    "    with tf.name_scope('hidden1'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([IMAGE_PIXELS, hidden1_units],\n",
    "                                stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([hidden1_units]), name='biases')\n",
    "        hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)\n",
    "\n",
    "    # Hidden 2\n",
    "    with tf.name_scope('hidden2'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([hidden1_units, hidden2_units],\n",
    "                                stddev=1.0 / math.sqrt(float(hidden1_units))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([hidden2_units]), name='biases')\n",
    "        hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases)\n",
    "\n",
    "    # Linear\n",
    "    with tf.name_scope('softmax_linear'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([hidden2_units, NUM_CLASSES],\n",
    "                                stddev=1.0 / math.sqrt(float(hidden2_units))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([NUM_CLASSES]), name='biases')\n",
    "        logits = tf.matmul(hidden2, weights) + biases\n",
    "        \n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_loss(logits, labels):\n",
    "    \"\"\"Calculates the loss from the logits and the labels.\n",
    "    Args:\n",
    "        logits: Logits tensor, float - [batch_size, NUM_CLASSES].\n",
    "        labels: Labels tensor, int32 - [batch_size].\n",
    "\n",
    "    Returns:\n",
    "        loss: Loss tensor of type float.\n",
    "    \"\"\"\n",
    "    labels = tf.to_int64(labels)\n",
    "    return tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_training(loss, learning_rate):\n",
    "    \"\"\"Sets up the training Ops.\n",
    "    \n",
    "    Creates a summarizer to track the loss over time in TensorBoard.\n",
    "    Creates an optimizer and applies the gradients to all trainable variables.\n",
    "    The Op returned by this function is what must be passed to the\n",
    "    `sess.run()` call to cause the model to train.\n",
    "\n",
    "    Args:\n",
    "        loss: Loss tensor, from loss().\n",
    "        learning_rate: The learning rate to use for gradient descent.\n",
    "    Returns:\n",
    "        train_op: The Op for training.\n",
    "      \"\"\"\n",
    "    # Add a scalar summary for the snapshot loss.\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    \n",
    "    # Create the gradient descent optimizer with the given learning rate.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    \n",
    "    # Create a variable to track the global step.\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    \n",
    "    # Use the optimizer to apply the gradients that minimize the loss\n",
    "    # (and also increment the global step counter) as a single training step.\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    return train_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_evaluation(logits, labels):\n",
    "    \"\"\"Evaluate the quality of the logits at predicting the label.\n",
    "\n",
    "    Args:\n",
    "        logits: Logits tensor, float - [batch_size, NUM_CLASSES].\n",
    "        labels: Labels tensor, int32 - [batch_size], with values in the range [0, NUM_CLASSES).\n",
    "\n",
    "    Returns:\n",
    "        A scalar int32 tensor with the number of examples (out of batch_size)\n",
    "        that were predicted correctly.\n",
    "    \"\"\"\n",
    "    # For a classifier model, we can use the in_top_k Op.\n",
    "    # It returns a bool tensor with shape [batch_size] that is true for\n",
    "    # the examples where the label is in the top k (here k=1)\n",
    "    # of all logits for that example.\n",
    "    correct = tf.nn.in_top_k(logits, labels, 1)\n",
    "\n",
    "    # Return the number of true entries.\n",
    "    return tf.reduce_sum(tf.cast(correct, tf.int32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trains and Evaluates the MNIST network using a feed dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def placeholder_inputs(batch_size):\n",
    "    \"\"\"Generate placeholder variables to represent the input tensors.\n",
    "\n",
    "    These placeholders are used as inputs by the rest of the model building\n",
    "    code and will be fed from the downloaded data in the .run() loop, below.\n",
    "\n",
    "    Args:\n",
    "        batch_size: The batch size will be baked into both placeholders.\n",
    "\n",
    "    Returns:\n",
    "        images_placeholder: Images placeholder.\n",
    "        labels_placeholder: Labels placeholder.\n",
    "    \"\"\"\n",
    "    # Note that the shapes of the placeholders match the shapes of the full\n",
    "    # image and label tensors, except the first dimension is now batch_size\n",
    "    # rather than the full size of the train or test data sets.\n",
    "    images_placeholder = tf.placeholder(tf.float32, shape=(batch_size, IMAGE_PIXELS))\n",
    "    labels_placeholder = tf.placeholder(tf.int32, shape=(batch_size))\n",
    "    return images_placeholder, labels_placeholder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_feed_dict(data_set, images_pl, labels_pl):\n",
    "    \"\"\"Fills the feed_dict for training the given step.\n",
    "\n",
    "    A feed_dict takes the form of:\n",
    "    feed_dict = {\n",
    "        <placeholder>: <tensor of values to be passed for placeholder>,\n",
    "          ....\n",
    "    }\n",
    "\n",
    "    Args:\n",
    "        data_set: The set of images and labels, from input_data.read_data_sets()\n",
    "        images_pl: The images placeholder, from placeholder_inputs().\n",
    "        labels_pl: The labels placeholder, from placeholder_inputs().\n",
    "\n",
    "    Returns:\n",
    "        feed_dict: The feed dictionary mapping from placeholders to values.\n",
    "    \"\"\"\n",
    "    # Create the feed_dict for the placeholders filled with the next\n",
    "    # `batch size` examples.\n",
    "    images_feed, labels_feed = data_set.next_batch(batch_size, fake_data)\n",
    "    feed_dict = {\n",
    "        images_pl: images_feed,\n",
    "        labels_pl: labels_feed,\n",
    "    }\n",
    "    return feed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_set):\n",
    "    \"\"\"Runs one evaluation against the full epoch of data.\n",
    "\n",
    "    Args:\n",
    "    sess: The session in which the model has been trained.\n",
    "    eval_correct: The Tensor that returns the number of correct predictions.\n",
    "    images_placeholder: The images placeholder.\n",
    "    labels_placeholder: The labels placeholder.\n",
    "    data_set: The set of images and labels to evaluate, from\n",
    "      input_data.read_data_sets().\n",
    "    \"\"\"\n",
    "    # And run one epoch of eval.\n",
    "    true_count = 0  # Counts the number of correct predictions.\n",
    "    steps_per_epoch = data_set.num_examples // batch_size\n",
    "    num_examples = steps_per_epoch * batch_size\n",
    "    for step in xrange(steps_per_epoch):\n",
    "        feed_dict = fill_feed_dict(data_set, images_placeholder, labels_placeholder)\n",
    "        true_count += sess.run(eval_correct, feed_dict=feed_dict)\n",
    "    precision = float(true_count) / num_examples\n",
    "    print('Num examples: %d  Num correct: %d  Precision @ 1: %0.04f' % (num_examples, true_count, precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_training():\n",
    "    \"\"\"Train MNIST for a number of steps.\n",
    "    \"\"\"\n",
    "    # Get the sets of images and labels for training, validation, and\n",
    "    # test on MNIST.\n",
    "    data_sets = input_data.read_data_sets(input_data_dir, fake_data)\n",
    "\n",
    "    # Tell TensorFlow that the model will be built into the default Graph.\n",
    "    with tf.Graph().as_default():\n",
    "        # Generate placeholders for the images and labels.\n",
    "        images_placeholder, labels_placeholder = placeholder_inputs(batch_size)\n",
    "\n",
    "        # Build a Graph that computes predictions from the inference model.\n",
    "        logits = mnist_inference(images_placeholder, hidden1, hidden2)\n",
    "\n",
    "        # Add to the Graph the Ops for loss calculation.\n",
    "        loss = mnist_loss(logits, labels_placeholder)\n",
    "    \n",
    "        # Quantization-aware training\n",
    "        if quantize_delay >= 0:\n",
    "            print('tf.contrib.quantize.create_training_graph')\n",
    "            tf.contrib.quantize.create_training_graph(quant_delay=quantize_delay)\n",
    "            \n",
    "        # Add to the Graph the Ops that calculate and apply gradients.\n",
    "        train_op = mnist_training(loss, learning_rate)\n",
    "\n",
    "        # Add the Op to compare the logits to the labels during evaluation.\n",
    "        eval_correct = mnist_evaluation(logits, labels_placeholder)\n",
    "\n",
    "        # Build the summary Tensor based on the TF collection of Summaries.\n",
    "        summary = tf.summary.merge_all()\n",
    "\n",
    "        # Add the variable initializer Op.\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "        # Create a saver for writing training checkpoints.\n",
    "        saver = tf.train.Saver()\n",
    "\n",
    "        # Create a session for running Ops on the Graph.\n",
    "        sess = tf.Session()\n",
    "\n",
    "        # Instantiate a SummaryWriter to output summaries and the Graph.\n",
    "        summary_writer = tf.summary.FileWriter(log_dir, sess.graph)\n",
    "\n",
    "        # And then after everything is built:\n",
    "\n",
    "        # Run the Op to initialize the variables.\n",
    "        sess.run(init)\n",
    "\n",
    "        # Start the training loop.\n",
    "        for step in xrange(max_steps):\n",
    "            start_time = time.time()\n",
    "\n",
    "            # Fill a feed dictionary with the actual set of images and labels\n",
    "            # for this particular training step.\n",
    "            feed_dict = fill_feed_dict(data_sets.train, images_placeholder, labels_placeholder)\n",
    "\n",
    "            # Run one step of the model.  The return values are the activations\n",
    "            # from the `train_op` (which is discarded) and the `loss` Op.  To\n",
    "            # inspect the values of your Ops or variables, you may include them\n",
    "            # in the list passed to sess.run() and the value tensors will be\n",
    "            # returned in the tuple from the call.\n",
    "            _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)\n",
    "\n",
    "            duration = time.time() - start_time\n",
    "\n",
    "            # Write the summaries and print an overview fairly often.\n",
    "            if step % 100 == 0:\n",
    "                # Print status to stdout.\n",
    "                print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))\n",
    "                # Update the events file.\n",
    "                summary_str = sess.run(summary, feed_dict=feed_dict)\n",
    "                summary_writer.add_summary(summary_str, step)\n",
    "                summary_writer.flush()\n",
    "\n",
    "            # Save a checkpoint and evaluate the model periodically.\n",
    "            if (step + 1) % 1000 == 0 or (step + 1) == max_steps:\n",
    "                checkpoint_file = os.path.join(log_dir, 'model.ckpt')\n",
    "                saver.save(sess, checkpoint_file, global_step=step)\n",
    "                # Evaluate against the training set.\n",
    "                print('Training Data Eval:')\n",
    "                do_eval(sess,\n",
    "                        eval_correct,\n",
    "                        images_placeholder,\n",
    "                        labels_placeholder,\n",
    "                        data_sets.train)\n",
    "                # Evaluate against the validation set.\n",
    "                print('Validation Data Eval:')\n",
    "                do_eval(sess,\n",
    "                        eval_correct,\n",
    "                        images_placeholder,\n",
    "                        labels_placeholder,\n",
    "                        data_sets.validation)\n",
    "                # Evaluate against the test set.\n",
    "                print('Test Data Eval:')\n",
    "                do_eval(sess,\n",
    "                        eval_correct,\n",
    "                        images_placeholder,\n",
    "                        labels_placeholder,\n",
    "                        data_sets.test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if tf.gfile.Exists(log_dir):\n",
    "    tf.gfile.DeleteRecursively(log_dir)\n",
    "tf.gfile.MakeDirs(log_dir)\n",
    "run_training()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saves out a GraphDef containing the architecture of the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.platform import gfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Graph().as_default() as graph:\n",
    "    # Generate placeholders for the images\n",
    "    placeholder = tf.placeholder(name='input_tensor', dtype=tf.float32, shape=(1, IMAGE_PIXELS))\n",
    "    \n",
    "    # Build a Graph that computes predictions from the inference model.\n",
    "    logits = mnist_inference(placeholder, hidden1, hidden2)\n",
    "\n",
    "    tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "\n",
    "    # Rewrites an eval input_graph in place for simulated quantization.\n",
    "    if quantize_delay >= 0:\n",
    "        tf.contrib.quantize.create_eval_graph()\n",
    "        \n",
    "    graph_def = graph.as_graph_def()\n",
    "    output_file = os.path.join(log_dir, 'mnist_fully_connected_feed_graph.pb')\n",
    "    with gfile.GFile(output_file, 'wb') as f:\n",
    "        f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "\n",
    "    model_filename = os.path.join(log_dir, 'mnist_fully_connected_feed_graph.pb')\n",
    "    with gfile.FastGFile(model_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "        g_in = tf.import_graph_def(graph_def)\n",
    "\n",
    "    LOGDIR = os.path.join(log_dir, 'GraphDef')\n",
    "    train_writer = tf.summary.FileWriter(LOGDIR)\n",
    "    train_writer.add_graph(sess.graph)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}