TF_simple_training_demo_final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First we'll define our query\n",
    "\n",
    "### Github Repo Dataset, includes files and metadata\n",
    "### Predict language given file contents\n",
    "### We want to find the suffix of each file, and make sure the content is at least 1024 characters long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s=r'''SELECT regexp_extract(sample_path, '.*(\\\\..+)$') as suffix,\n",
    "             content\n",
    "      FROM  [bigquery-public-data:github_repos.sample_contents]      \n",
    "      where \n",
    "          content IS NOT NULL\n",
    "          AND content != ''\n",
    "          and length(content) > 1024\n",
    "      HAVING \n",
    "             suffix IS NOT NULL\n",
    "             AND suffix in ('.py','.c','rb')\n",
    "      LIMIT 1000;'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Next we'll obtain the default Big Query client and run the query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from google.cloud import bigquery\n",
    "c = bigquery.Client()\n",
    "query = c.run_sync_query(s)\n",
    "query.run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We then want to transform our data to base 64, control size, and write to a csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "from base64 import urlsafe_b64encode\n",
    "\n",
    "\n",
    "def transform_content(content):\n",
    "    content = content.encode('utf-8')    \n",
    "    content = content[:1024]    \n",
    "    return content\n",
    "\n",
    "with file('/tmp/data.csv','wb') as out_file:\n",
    "    w = csv.writer(out_file, quoting=csv.QUOTE_ALL, delimiter='\\t')\n",
    "    for language,content in query.rows:\n",
    "        w.writerow([language,urlsafe_b64encode(transform_content(content))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import and set constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 0.01\n",
    "TRAIN_ITERATIONS = 1000\n",
    "MAX_STRING_SIZE = 1024"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start reading data into queue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames_queue = tf.train.string_input_producer(['/tmp/data.csv'])\n",
    "reader = tf.TextLineReader()\n",
    "key, value = reader.read(filenames_queue)\n",
    "default_values=[['UNKNOWN'], ['']]# decode content\n",
    "language, b64_content = tf.decode_csv(value,default_values, field_delim='\\t')\n",
    "content = tf.decode_base64(b64_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create batches from queus by shuffling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "language_batch_op, content_batch_op = tf.train.shuffle_batch([language,content], \n",
    "                                                             batch_size=BATCH_SIZE, \n",
    "                                                             capacity=1000, \n",
    "                                                             min_after_dequeue=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make hash table and one hot encoding for langauges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "language_keys=['.py','.c','.rb']\n",
    "values=range(1, len(language_keys)+1)\n",
    "language_codes_table = tf.contrib.lookup.HashTable(\n",
    "    tf.contrib.lookup.KeyValueTensorInitializer(language_keys, values), 0)\n",
    "\n",
    "language_codes_indices = language_codes_table.lookup(language_batch_op)\n",
    "language_codes_batch_op = tf.one_hot(language_codes_indices, len(language_keys))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make embeddings for the characters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bytes = tf.transpose(tf.decode_raw(content_batch_op, tf.uint8))\n",
    "bytes_embedding_weights = tf.Variable(name=\"embedding_weights\",\n",
    "                                      initial_value=tf.random_uniform(shape=(256, 64),\n",
    "                                                                      minval=-0.1, \n",
    "                                                                      maxval=0.1))\n",
    "bytes_embedding = tf.nn.embedding_lookup(bytes_embedding_weights, tf.cast(bytes,tf.int32))\n",
    "embedding_mean = tf.reduce_mean(bytes_embedding,axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dense layer with Relu and bias, from embedding length to number of categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dense_weights = tf.get_variable(name='dense_weights',\n",
    "                          shape=[64, len(language_keys)],\n",
    "                          initializer=tf.contrib.layers.xavier_initializer())\n",
    "biases = tf.Variable(tf.zeros([len(language_keys)]), name='biases')\n",
    "logits = tf.nn.relu(tf.matmul(embedding_mean, dense_weights) + biases, name='logits')\n",
    "prediction = tf.argmax(logits, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_loss = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=language_codes_batch_op)\n",
    "loss_op = tf.reduce_mean(batch_loss, name='loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define optimizer, global step counter, and train operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)\n",
    "global_step = tf.Variable(0,name='global_step', trainable=False)\n",
    "train_op = optimizer.minimize(loss_op,global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize variables, tables, and session "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_vars_op = tf.global_variables_initializer()\n",
    "init_tables_op = tf.tables_initializer()\n",
    "\n",
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We will now start the session, using a train Coordinator and Queue runners\n",
    "### We'll run for all the train iterations and then stop all threads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 1.09998\n",
      "Train loss: 1.08718\n",
      "Train loss: 1.06466\n",
      "Train loss: 1.03933\n",
      "Train loss: 1.02273\n",
      "Train loss: 1.00025\n",
      "Train loss: 0.961968\n",
      "Train loss: 0.939344\n",
      "Train loss: 0.905729\n",
      "Train loss: 0.890576\n",
      "Train loss: 0.849384\n",
      "Train loss: 0.833741\n",
      "Train loss: 0.803506\n",
      "Train loss: 0.795069\n",
      "Train loss: 0.752052\n",
      "Train loss: 0.748417\n",
      "Train loss: 0.721226\n",
      "Train loss: 0.700206\n",
      "Train loss: 0.670752\n",
      "Train loss: 0.64038\n",
      "Train loss: 0.641447\n",
      "Train loss: 0.631414\n",
      "Train loss: 0.596992\n",
      "Train loss: 0.575163\n",
      "Train loss: 0.544281\n",
      "Train loss: 0.552259\n",
      "Train loss: 0.54449\n",
      "Train loss: 0.522265\n",
      "Train loss: 0.515425\n",
      "Train loss: 0.484299\n",
      "Train loss: 0.472563\n",
      "Train loss: 0.460844\n",
      "Train loss: 0.469132\n",
      "Train loss: 0.457314\n",
      "Train loss: 0.409448\n",
      "Train loss: 0.437277\n",
      "Train loss: 0.404841\n",
      "Train loss: 0.404376\n",
      "Train loss: 0.370697\n",
      "Train loss: 0.3507\n",
      "Train loss: 0.36521\n",
      "Train loss: 0.375835\n",
      "Train loss: 0.326528\n",
      "Train loss: 0.355624\n",
      "Train loss: 0.320836\n",
      "Train loss: 0.316329\n",
      "Train loss: 0.285463\n",
      "Train loss: 0.299892\n",
      "Train loss: 0.292902\n",
      "Train loss: 0.31078\n",
      "Train loss: 0.244396\n",
      "Train loss: 0.289736\n",
      "Train loss: 0.270982\n",
      "Train loss: 0.252948\n",
      "Train loss: 0.247118\n",
      "Train loss: 0.278325\n",
      "Train loss: 0.236422\n",
      "Train loss: 0.230069\n",
      "Train loss: 0.23961\n",
      "Train loss: 0.221089\n",
      "Train loss: 0.243323\n",
      "Train loss: 0.217347\n",
      "Train loss: 0.214069\n",
      "Train loss: 0.203478\n",
      "Train loss: 0.209182\n",
      "Train loss: 0.186671\n",
      "Train loss: 0.208609\n",
      "Train loss: 0.17809\n",
      "Train loss: 0.167485\n",
      "Train loss: 0.175357\n",
      "Train loss: 0.169788\n",
      "Train loss: 0.19165\n",
      "Train loss: 0.178727\n",
      "Train loss: 0.167936\n",
      "Train loss: 0.166848\n",
      "Train loss: 0.174069\n",
      "Train loss: 0.199818\n",
      "Train loss: 0.179018\n",
      "Train loss: 0.127035\n",
      "Train loss: 0.158989\n",
      "Train loss: 0.128265\n",
      "Train loss: 0.136054\n",
      "Train loss: 0.142835\n",
      "Train loss: 0.150439\n",
      "Train loss: 0.131962\n",
      "Train loss: 0.159413\n",
      "Train loss: 0.126682\n",
      "Train loss: 0.139109\n",
      "Train loss: 0.145927\n",
      "Train loss: 0.1324\n",
      "Train loss: 0.135859\n",
      "Train loss: 0.118037\n",
      "Train loss: 0.144227\n",
      "Train loss: 0.119217\n",
      "Train loss: 0.117124\n",
      "Train loss: 0.130599\n",
      "Train loss: 0.10606\n",
      "Train loss: 0.131009\n",
      "Train loss: 0.143709\n",
      "Train loss: 0.109133\n",
      "Train loss: 0.0668848\n",
      "Train loss: 0.0881283\n",
      "Train loss: 0.113678\n",
      "Train loss: 0.0974661\n",
      "Train loss: 0.0879611\n",
      "Train loss: 0.114561\n",
      "Train loss: 0.0984392\n",
      "Train loss: 0.0808612\n",
      "Train loss: 0.147281\n",
      "Train loss: 0.100165\n",
      "Train loss: 0.125761\n",
      "Train loss: 0.13766\n",
      "Train loss: 0.0779183\n",
      "Train loss: 0.0706343\n",
      "Train loss: 0.072454\n",
      "Train loss: 0.107409\n",
      "Train loss: 0.136497\n",
      "Train loss: 0.0880271\n",
      "Train loss: 0.0878691\n",
      "Train loss: 0.107893\n",
      "Train loss: 0.0938777\n",
      "Train loss: 0.109719\n",
      "Train loss: 0.112929\n",
      "Train loss: 0.0578401\n",
      "Train loss: 0.069875\n",
      "Train loss: 0.070125\n",
      "Train loss: 0.0812791\n",
      "Train loss: 0.101319\n",
      "Train loss: 0.0759016\n",
      "Train loss: 0.0942208\n",
      "Train loss: 0.107359\n",
      "Train loss: 0.0609918\n",
      "Train loss: 0.0795859\n",
      "Train loss: 0.112614\n",
      "Train loss: 0.0585848\n",
      "Train loss: 0.087874\n",
      "Train loss: 0.0784875\n",
      "Train loss: 0.0677464\n",
      "Train loss: 0.0680351\n",
      "Train loss: 0.0986238\n",
      "Train loss: 0.0711456\n",
      "Train loss: 0.0707692\n",
      "Train loss: 0.0718431\n",
      "Train loss: 0.0591145\n",
      "Train loss: 0.0516787\n",
      "Train loss: 0.0792389\n",
      "Train loss: 0.066286\n",
      "Train loss: 0.0775704\n",
      "Train loss: 0.0689513\n",
      "Train loss: 0.0618889\n",
      "Train loss: 0.0618164\n",
      "Train loss: 0.0855541\n",
      "Train loss: 0.0734963\n",
      "Train loss: 0.0622096\n",
      "Train loss: 0.0531886\n",
      "Train loss: 0.0678406\n",
      "Train loss: 0.0647784\n",
      "Train loss: 0.071917\n",
      "Train loss: 0.0523169\n",
      "Train loss: 0.0911643\n",
      "Train loss: 0.0585384\n",
      "Train loss: 0.0439208\n",
      "Train loss: 0.065796\n",
      "Train loss: 0.0773128\n",
      "Train loss: 0.0524652\n",
      "Train loss: 0.060114\n",
      "Train loss: 0.0596281\n",
      "Train loss: 0.0635886\n",
      "Train loss: 0.0466653\n",
      "Train loss: 0.0359311\n",
      "Train loss: 0.0640134\n",
      "Train loss: 0.0443261\n",
      "Train loss: 0.0662913\n",
      "Train loss: 0.0397483\n",
      "Train loss: 0.0558137\n",
      "Train loss: 0.0400376\n",
      "Train loss: 0.0426884\n",
      "Train loss: 0.080609\n",
      "Train loss: 0.0442154\n",
      "Train loss: 0.0436672\n",
      "Train loss: 0.0464108\n",
      "Train loss: 0.045437\n",
      "Train loss: 0.0634313\n",
      "Train loss: 0.0438775\n",
      "Train loss: 0.0605602\n",
      "Train loss: 0.0448234\n",
      "Train loss: 0.035921\n",
      "Train loss: 0.0333088\n",
      "Train loss: 0.0621518\n",
      "Train loss: 0.07633\n",
      "Train loss: 0.0536393\n",
      "Train loss: 0.0518999\n",
      "Train loss: 0.046133\n",
      "Train loss: 0.0583119\n",
      "Train loss: 0.0529729\n",
      "Train loss: 0.0305879\n",
      "Train loss: 0.0419159\n",
      "Train loss: 0.0306338\n",
      "Train loss: 0.0513852\n",
      "Train loss: 0.0476203\n",
      "Train loss: 0.0379362\n",
      "Train loss: 0.0365561\n",
      "Train loss: 0.0475139\n",
      "Train loss: 0.034364\n",
      "Train loss: 0.0471968\n",
      "Train loss: 0.041114\n",
      "Train loss: 0.0374459\n",
      "Train loss: 0.0580833\n",
      "Train loss: 0.0425314\n",
      "Train loss: 0.0243326\n",
      "Train loss: 0.04284\n",
      "Train loss: 0.0278073\n",
      "Train loss: 0.0388309\n",
      "Train loss: 0.0293598\n",
      "Train loss: 0.0327455\n",
      "Train loss: 0.0212818\n",
      "Train loss: 0.0365849\n",
      "Train loss: 0.0236482\n",
      "Train loss: 0.0241854\n",
      "Train loss: 0.060124\n",
      "Train loss: 0.0314953\n",
      "Train loss: 0.0171783\n",
      "Train loss: 0.0364784\n",
      "Train loss: 0.05564\n",
      "Train loss: 0.0307441\n",
      "Train loss: 0.0338315\n",
      "Train loss: 0.0340197\n",
      "Train loss: 0.0429815\n",
      "Train loss: 0.0337066\n",
      "Train loss: 0.0425215\n",
      "Train loss: 0.0199775\n",
      "Train loss: 0.0306609\n",
      "Train loss: 0.0447783\n",
      "Train loss: 0.022983\n",
      "Train loss: 0.0261586\n",
      "Train loss: 0.0341304\n",
      "Train loss: 0.0279227\n",
      "Train loss: 0.0258644\n",
      "Train loss: 0.025413\n",
      "Train loss: 0.0260951\n",
      "Train loss: 0.0338882\n",
      "Train loss: 0.0303136\n",
      "Train loss: 0.0434551\n",
      "Train loss: 0.0283388\n",
      "Train loss: 0.0592909\n",
      "Train loss: 0.0291201\n",
      "Train loss: 0.0281781\n",
      "Train loss: 0.0614269\n",
      "Train loss: 0.0479404\n",
      "Train loss: 0.0270632\n",
      "Train loss: 0.0259908\n",
      "Train loss: 0.0193312\n",
      "Train loss: 0.0276409\n",
      "Train loss: 0.0304984\n",
      "Train loss: 0.0343194\n",
      "Train loss: 0.0431377\n",
      "Train loss: 0.0251434\n",
      "Train loss: 0.0318212\n",
      "Train loss: 0.0150354\n",
      "Train loss: 0.0282311\n",
      "Train loss: 0.0333374\n",
      "Train loss: 0.0273845\n",
      "Train loss: 0.0263631\n",
      "Train loss: 0.0271715\n",
      "Train loss: 0.0309113\n",
      "Train loss: 0.0319937\n",
      "Train loss: 0.0263757\n",
      "Train loss: 0.0242085\n",
      "Train loss: 0.029887\n",
      "Train loss: 0.0326232\n",
      "Train loss: 0.0206854\n",
      "Train loss: 0.0156575\n",
      "Train loss: 0.0221149\n",
      "Train loss: 0.0174746\n",
      "Train loss: 0.0349288\n",
      "Train loss: 0.0255656\n",
      "Train loss: 0.0182399\n",
      "Train loss: 0.0305023\n",
      "Train loss: 0.0277726\n",
      "Train loss: 0.0233671\n",
      "Train loss: 0.017029\n",
      "Train loss: 0.0263793\n",
      "Train loss: 0.0216587\n",
      "Train loss: 0.0263785\n",
      "Train loss: 0.0433276\n",
      "Train loss: 0.0268833\n",
      "Train loss: 0.0243194\n",
      "Train loss: 0.0221413\n",
      "Train loss: 0.0162119\n",
      "Train loss: 0.0207705\n",
      "Train loss: 0.0206161\n",
      "Train loss: 0.0314839\n",
      "Train loss: 0.0347034\n",
      "Train loss: 0.0233668\n",
      "Train loss: 0.0182234\n",
      "Train loss: 0.0155337\n",
      "Train loss: 0.0205722\n",
      "Train loss: 0.0152556\n",
      "Train loss: 0.0277379\n",
      "Train loss: 0.0158988\n",
      "Train loss: 0.0226571\n",
      "Train loss: 0.0295006\n",
      "Train loss: 0.0183025\n",
      "Train loss: 0.0136544\n",
      "Train loss: 0.0320791\n",
      "Train loss: 0.0129719\n",
      "Train loss: 0.0291082\n",
      "Train loss: 0.0230713\n",
      "Train loss: 0.0193914\n",
      "Train loss: 0.0176398\n",
      "Train loss: 0.019276\n",
      "Train loss: 0.0328111\n",
      "Train loss: 0.0141567\n",
      "Train loss: 0.0210131\n",
      "Train loss: 0.0140044\n",
      "Train loss: 0.0125459\n",
      "Train loss: 0.0250862\n",
      "Train loss: 0.0283578\n",
      "Train loss: 0.0114199\n",
      "Train loss: 0.0177086\n",
      "Train loss: 0.0156735\n",
      "Train loss: 0.0256521\n",
      "Train loss: 0.0287058\n",
      "Train loss: 0.0204016\n",
      "Train loss: 0.025\n",
      "Train loss: 0.0215643\n",
      "Train loss: 0.0193687\n",
      "Train loss: 0.0228955\n",
      "Train loss: 0.0180858\n",
      "Train loss: 0.0164075\n",
      "Train loss: 0.0151173\n",
      "Train loss: 0.015197\n",
      "Train loss: 0.0171159\n",
      "Train loss: 0.0224866\n",
      "Train loss: 0.0232018\n",
      "Train loss: 0.0148634\n",
      "Train loss: 0.029027\n",
      "Train loss: 0.0240003\n",
      "Train loss: 0.0107667\n",
      "Train loss: 0.0154566\n",
      "Train loss: 0.0130678\n",
      "Train loss: 0.0162156\n",
      "Train loss: 0.0182939\n",
      "Train loss: 0.0281053\n",
      "Train loss: 0.0113585\n",
      "Train loss: 0.0146805\n",
      "Train loss: 0.0160971\n",
      "Train loss: 0.0121692\n",
      "Train loss: 0.0271676\n",
      "Train loss: 0.0225216\n",
      "Train loss: 0.00777164\n",
      "Train loss: 0.0184335\n",
      "Train loss: 0.0130245\n",
      "Train loss: 0.0149912\n",
      "Train loss: 0.0246919\n",
      "Train loss: 0.0133853\n",
      "Train loss: 0.0130563\n",
      "Train loss: 0.013661\n",
      "Train loss: 0.0211179\n",
      "Train loss: 0.0209383\n",
      "Train loss: 0.0151281\n",
      "Train loss: 0.00967335\n",
      "Train loss: 0.0105353\n",
      "Train loss: 0.0199727\n",
      "Train loss: 0.0182715\n",
      "Train loss: 0.0179212\n",
      "Train loss: 0.0129635\n",
      "Train loss: 0.00970237\n",
      "Train loss: 0.0158863\n",
      "Train loss: 0.0134588\n",
      "Train loss: 0.0270035\n",
      "Train loss: 0.00969705\n",
      "Train loss: 0.0176263\n",
      "Train loss: 0.0126533\n",
      "Train loss: 0.0243728\n",
      "Train loss: 0.0180524\n",
      "Train loss: 0.0241161\n",
      "Train loss: 0.0178165\n",
      "Train loss: 0.0165937\n",
      "Train loss: 0.00912206\n",
      "Train loss: 0.0144311\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.0143603\n",
      "Train loss: 0.0197304\n",
      "Train loss: 0.0128187\n",
      "Train loss: 0.0106895\n",
      "Train loss: 0.0228384\n",
      "Train loss: 0.023421\n",
      "Train loss: 0.0128262\n",
      "Train loss: 0.0115354\n",
      "Train loss: 0.0125654\n",
      "Train loss: 0.0112655\n",
      "Train loss: 0.0137601\n",
      "Train loss: 0.0185784\n",
      "Train loss: 0.00765675\n",
      "Train loss: 0.0102258\n",
      "Train loss: 0.0154883\n",
      "Train loss: 0.01793\n",
      "Train loss: 0.0136768\n",
      "Train loss: 0.0125636\n",
      "Train loss: 0.0147032\n",
      "Train loss: 0.0153072\n",
      "Train loss: 0.0100479\n",
      "Train loss: 0.014395\n",
      "Train loss: 0.0111869\n",
      "Train loss: 0.0129578\n",
      "Train loss: 0.0210022\n",
      "Train loss: 0.0104114\n",
      "Train loss: 0.0172104\n",
      "Train loss: 0.016571\n",
      "Train loss: 0.0199602\n",
      "Train loss: 0.0180722\n",
      "Train loss: 0.0118401\n",
      "Train loss: 0.0114885\n",
      "Train loss: 0.011522\n",
      "Train loss: 0.0101551\n",
      "Train loss: 0.00919563\n",
      "Train loss: 0.0123015\n",
      "Train loss: 0.0134097\n",
      "Train loss: 0.0200256\n",
      "Train loss: 0.0146669\n",
      "Train loss: 0.00771676\n",
      "Train loss: 0.0159156\n",
      "Train loss: 0.0146457\n",
      "Train loss: 0.00746825\n",
      "Train loss: 0.0100763\n",
      "Train loss: 0.00802957\n",
      "Train loss: 0.0190307\n",
      "Train loss: 0.0191348\n",
      "Train loss: 0.00582213\n",
      "Train loss: 0.0129152\n",
      "Train loss: 0.0111296\n",
      "Train loss: 0.0246424\n",
      "Train loss: 0.0100293\n",
      "Train loss: 0.00631369\n",
      "Train loss: 0.0122215\n",
      "Train loss: 0.00817957\n",
      "Train loss: 0.0171778\n",
      "Train loss: 0.0110189\n",
      "Train loss: 0.00634027\n",
      "Train loss: 0.0196989\n",
      "Train loss: 0.0106248\n",
      "Train loss: 0.0104544\n",
      "Train loss: 0.0180776\n",
      "Train loss: 0.0159482\n",
      "Train loss: 0.00836842\n",
      "Train loss: 0.0206924\n",
      "Train loss: 0.00935212\n",
      "Train loss: 0.00787903\n",
      "Train loss: 0.00946347\n",
      "Train loss: 0.0106934\n",
      "Train loss: 0.00825239\n",
      "Train loss: 0.00620255\n",
      "Train loss: 0.0132572\n",
      "Train loss: 0.0154613\n",
      "Train loss: 0.0127858\n",
      "Train loss: 0.00917294\n",
      "Train loss: 0.017229\n",
      "Train loss: 0.0221521\n",
      "Train loss: 0.0111046\n",
      "Train loss: 0.0107217\n",
      "Train loss: 0.00848446\n",
      "Train loss: 0.0178288\n",
      "Train loss: 0.0115945\n",
      "Train loss: 0.00833561\n",
      "Train loss: 0.0122021\n",
      "Train loss: 0.00489102\n",
      "Train loss: 0.00669114\n",
      "Train loss: 0.0123593\n",
      "Train loss: 0.0111725\n",
      "Train loss: 0.0050971\n",
      "Train loss: 0.0077405\n",
      "Train loss: 0.0151194\n",
      "Train loss: 0.017728\n",
      "Train loss: 0.0122198\n",
      "Train loss: 0.00834514\n",
      "Train loss: 0.0188713\n",
      "Train loss: 0.00664077\n",
      "Train loss: 0.00886295\n",
      "Train loss: 0.00903107\n",
      "Train loss: 0.0183249\n",
      "Train loss: 0.00569947\n",
      "Train loss: 0.00828977\n",
      "Train loss: 0.0123556\n",
      "Train loss: 0.0115998\n",
      "Train loss: 0.0108101\n",
      "Train loss: 0.00687587\n",
      "Train loss: 0.0123555\n",
      "Train loss: 0.0110168\n",
      "Train loss: 0.00727887\n",
      "Train loss: 0.00662343\n",
      "Train loss: 0.00984658\n",
      "Train loss: 0.0079781\n",
      "Train loss: 0.00787195\n",
      "Train loss: 0.00775928\n",
      "Train loss: 0.0120234\n",
      "Train loss: 0.0239591\n",
      "Train loss: 0.00882735\n",
      "Train loss: 0.00820098\n",
      "Train loss: 0.00653529\n",
      "Train loss: 0.0113754\n",
      "Train loss: 0.0156336\n",
      "Train loss: 0.00759617\n",
      "Train loss: 0.0152105\n",
      "Train loss: 0.0110106\n",
      "Train loss: 0.00990325\n",
      "Train loss: 0.0121642\n",
      "Train loss: 0.00531133\n",
      "Train loss: 0.00776972\n",
      "Train loss: 0.0126644\n",
      "Train loss: 0.00882851\n",
      "Train loss: 0.00612289\n",
      "Train loss: 0.00734313\n",
      "Train loss: 0.017404\n",
      "Train loss: 0.00859005\n",
      "Train loss: 0.00493386\n",
      "Train loss: 0.00568038\n",
      "Train loss: 0.0105849\n",
      "Train loss: 0.0123619\n",
      "Train loss: 0.019336\n",
      "Train loss: 0.00691779\n",
      "Train loss: 0.0144238\n",
      "Train loss: 0.00572056\n",
      "Train loss: 0.00714031\n",
      "Train loss: 0.00446504\n",
      "Train loss: 0.00838725\n",
      "Train loss: 0.0107901\n",
      "Train loss: 0.00848242\n",
      "Train loss: 0.00354557\n",
      "Train loss: 0.00468458\n",
      "Train loss: 0.00525993\n",
      "Train loss: 0.0101554\n",
      "Train loss: 0.00585005\n",
      "Train loss: 0.0134792\n",
      "Train loss: 0.00732803\n",
      "Train loss: 0.0049193\n",
      "Train loss: 0.00996848\n",
      "Train loss: 0.0100621\n",
      "Train loss: 0.0125353\n",
      "Train loss: 0.00458599\n",
      "Train loss: 0.0149005\n",
      "Train loss: 0.0102585\n",
      "Train loss: 0.00743029\n",
      "Train loss: 0.0105975\n",
      "Train loss: 0.00583629\n",
      "Train loss: 0.0110558\n",
      "Train loss: 0.00751908\n",
      "Train loss: 0.0152516\n",
      "Train loss: 0.0191378\n",
      "Train loss: 0.00531554\n",
      "Train loss: 0.011791\n",
      "Train loss: 0.00835231\n",
      "Train loss: 0.00930404\n",
      "Train loss: 0.00592547\n",
      "Train loss: 0.00894487\n",
      "Train loss: 0.00507458\n",
      "Train loss: 0.00671788\n",
      "Train loss: 0.00772764\n",
      "Train loss: 0.00786073\n",
      "Train loss: 0.00418163\n",
      "Train loss: 0.0139374\n",
      "Train loss: 0.00639911\n",
      "Train loss: 0.00574065\n",
      "Train loss: 0.00485721\n",
      "Train loss: 0.0103518\n",
      "Train loss: 0.010445\n",
      "Train loss: 0.0132606\n",
      "Train loss: 0.0112816\n",
      "Train loss: 0.00442207\n",
      "Train loss: 0.00477597\n",
      "Train loss: 0.00413533\n",
      "Train loss: 0.0142355\n",
      "Train loss: 0.00647473\n",
      "Train loss: 0.00788531\n",
      "Train loss: 0.00893833\n",
      "Train loss: 0.00946681\n",
      "Train loss: 0.00735868\n",
      "Train loss: 0.00981969\n",
      "Train loss: 0.00577883\n",
      "Train loss: 0.010371\n",
      "Train loss: 0.0099142\n",
      "Train loss: 0.00402209\n",
      "Train loss: 0.0057682\n",
      "Train loss: 0.00971117\n",
      "Train loss: 0.00632111\n",
      "Train loss: 0.0064031\n",
      "Train loss: 0.00537752\n",
      "Train loss: 0.00933233\n",
      "Train loss: 0.0095952\n",
      "Train loss: 0.00751242\n",
      "Train loss: 0.0080945\n",
      "Train loss: 0.00622331\n",
      "Train loss: 0.00556551\n",
      "Train loss: 0.00734644\n",
      "Train loss: 0.00379455\n",
      "Train loss: 0.00824081\n",
      "Train loss: 0.00565015\n",
      "Train loss: 0.0098604\n",
      "Train loss: 0.0096329\n",
      "Train loss: 0.0074909\n",
      "Train loss: 0.00528211\n",
      "Train loss: 0.00770919\n",
      "Train loss: 0.00457944\n",
      "Train loss: 0.00916784\n",
      "Train loss: 0.00433568\n",
      "Train loss: 0.0130911\n",
      "Train loss: 0.0145431\n",
      "Train loss: 0.00592568\n",
      "Train loss: 0.00530226\n",
      "Train loss: 0.0216883\n",
      "Train loss: 0.00530764\n",
      "Train loss: 0.00607916\n",
      "Train loss: 0.010944\n",
      "Train loss: 0.00485636\n",
      "Train loss: 0.00514932\n",
      "Train loss: 0.00614459\n",
      "Train loss: 0.0082489\n",
      "Train loss: 0.006949\n",
      "Train loss: 0.00656345\n",
      "Train loss: 0.0206482\n",
      "Train loss: 0.00300839\n",
      "Train loss: 0.0073502\n",
      "Train loss: 0.00587072\n",
      "Train loss: 0.0130924\n",
      "Train loss: 0.00886648\n",
      "Train loss: 0.00649877\n",
      "Train loss: 0.00995114\n",
      "Train loss: 0.00604194\n",
      "Train loss: 0.00369915\n",
      "Train loss: 0.0086555\n",
      "Train loss: 0.0101757\n",
      "Train loss: 0.00312274\n",
      "Train loss: 0.00283342\n",
      "Train loss: 0.00704125\n",
      "Train loss: 0.0080388\n",
      "Train loss: 0.00614325\n",
      "Train loss: 0.00805296\n",
      "Train loss: 0.00902415\n",
      "Train loss: 0.00391658\n",
      "Train loss: 0.00534707\n",
      "Train loss: 0.00847021\n",
      "Train loss: 0.00306361\n",
      "Train loss: 0.00416544\n",
      "Train loss: 0.0032261\n",
      "Train loss: 0.0135528\n",
      "Train loss: 0.00618213\n",
      "Train loss: 0.00461467\n",
      "Train loss: 0.0108972\n",
      "Train loss: 0.00953595\n",
      "Train loss: 0.0042998\n",
      "Train loss: 0.0120813\n",
      "Train loss: 0.0058334\n",
      "Train loss: 0.00499837\n",
      "Train loss: 0.00668003\n",
      "Train loss: 0.00448502\n",
      "Train loss: 0.00468327\n",
      "Train loss: 0.00556799\n",
      "Train loss: 0.0108792\n",
      "Train loss: 0.00335511\n",
      "Train loss: 0.00381167\n",
      "Train loss: 0.00362115\n",
      "Train loss: 0.00464045\n",
      "Train loss: 0.006539\n",
      "Train loss: 0.00762753\n",
      "Train loss: 0.00455475\n",
      "Train loss: 0.00488884\n",
      "Train loss: 0.0039079\n",
      "Train loss: 0.0115104\n",
      "Train loss: 0.00873182\n",
      "Train loss: 0.00507325\n",
      "Train loss: 0.00382285\n",
      "Train loss: 0.0109266\n",
      "Train loss: 0.00459395\n",
      "Train loss: 0.0051176\n",
      "Train loss: 0.00287766\n",
      "Train loss: 0.00458308\n",
      "Train loss: 0.00431311\n",
      "Train loss: 0.00696534\n",
      "Train loss: 0.00331981\n",
      "Train loss: 0.00315426\n",
      "Train loss: 0.00369537\n",
      "Train loss: 0.00690624\n",
      "Train loss: 0.0103362\n",
      "Train loss: 0.00743842\n",
      "Train loss: 0.00388385\n",
      "Train loss: 0.0124669\n",
      "Train loss: 0.00484715\n",
      "Train loss: 0.0067446\n",
      "Train loss: 0.00314397\n",
      "Train loss: 0.00260265\n",
      "Train loss: 0.00674444\n",
      "Train loss: 0.0100585\n",
      "Train loss: 0.00485385\n",
      "Train loss: 0.0102593\n",
      "Train loss: 0.00854749\n",
      "Train loss: 0.00907272\n",
      "Train loss: 0.00479781\n",
      "Train loss: 0.00796256\n",
      "Train loss: 0.00336565\n",
      "Train loss: 0.00346349\n",
      "Train loss: 0.0103674\n",
      "Train loss: 0.00472408\n",
      "Train loss: 0.0042108\n",
      "Train loss: 0.00502538\n",
      "Train loss: 0.00363869\n",
      "Train loss: 0.00412787\n",
      "Train loss: 0.00869992\n",
      "Train loss: 0.00852637\n",
      "Train loss: 0.00731587\n",
      "Train loss: 0.00538474\n",
      "Train loss: 0.00187745\n",
      "Train loss: 0.00500986\n",
      "Train loss: 0.00648559\n",
      "Train loss: 0.00286187\n",
      "Train loss: 0.00315156\n",
      "Train loss: 0.00421117\n",
      "Train loss: 0.00273973\n",
      "Train loss: 0.00226693\n",
      "Train loss: 0.00697649\n",
      "Train loss: 0.00388558\n",
      "Train loss: 0.00897683\n",
      "Train loss: 0.00339171\n",
      "Train loss: 0.00378642\n",
      "Train loss: 0.00392405\n",
      "Train loss: 0.00540196\n",
      "Train loss: 0.00472685\n",
      "Train loss: 0.00384613\n",
      "Train loss: 0.00426885\n",
      "Train loss: 0.00624709\n",
      "Train loss: 0.00326653\n",
      "Train loss: 0.00261104\n",
      "Train loss: 0.0040575\n",
      "Train loss: 0.0051889\n",
      "Train loss: 0.00700871\n",
      "Train loss: 0.00606983\n",
      "Train loss: 0.00295238\n",
      "Train loss: 0.00653966\n",
      "Train loss: 0.00486606\n",
      "Train loss: 0.0053195\n",
      "Train loss: 0.00983032\n",
      "Train loss: 0.00358035\n",
      "Train loss: 0.0044298\n",
      "Train loss: 0.00415473\n",
      "Train loss: 0.00397195\n",
      "Train loss: 0.00913795\n",
      "Train loss: 0.0039186\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.00869631\n",
      "Train loss: 0.00673452\n",
      "Train loss: 0.00493908\n",
      "Train loss: 0.00608286\n",
      "Train loss: 0.00320008\n",
      "Train loss: 0.0054401\n",
      "Train loss: 0.0028852\n",
      "Train loss: 0.00893378\n",
      "Train loss: 0.0054515\n",
      "Train loss: 0.00390156\n",
      "Train loss: 0.00480551\n",
      "Train loss: 0.00687917\n",
      "Train loss: 0.00385056\n",
      "Train loss: 0.00597552\n",
      "Train loss: 0.000951224\n",
      "Train loss: 0.00816755\n",
      "Train loss: 0.0028105\n",
      "Train loss: 0.00340622\n",
      "Train loss: 0.00538887\n",
      "Train loss: 0.00676869\n",
      "Train loss: 0.00209056\n",
      "Train loss: 0.00134794\n",
      "Train loss: 0.00764507\n",
      "Train loss: 0.0045591\n",
      "Train loss: 0.00386891\n",
      "Train loss: 0.00298083\n",
      "Train loss: 0.00281659\n",
      "Train loss: 0.00425498\n",
      "Train loss: 0.00271393\n",
      "Train loss: 0.00735442\n",
      "Train loss: 0.0047339\n",
      "Train loss: 0.00501415\n",
      "Train loss: 0.0045275\n",
      "Train loss: 0.00195174\n",
      "Train loss: 0.00729796\n",
      "Train loss: 0.0066807\n",
      "Train loss: 0.00175155\n",
      "Train loss: 0.00292856\n",
      "Train loss: 0.00667825\n",
      "Train loss: 0.00396888\n",
      "Train loss: 0.00440776\n",
      "Train loss: 0.00438817\n",
      "Train loss: 0.00329923\n",
      "Train loss: 0.00249508\n",
      "Train loss: 0.00564714\n",
      "Train loss: 0.0060519\n",
      "Train loss: 0.00302984\n",
      "Train loss: 0.00757876\n",
      "Train loss: 0.00330611\n",
      "Train loss: 0.00309876\n",
      "Train loss: 0.00521597\n",
      "Train loss: 0.00479248\n",
      "Train loss: 0.00455402\n",
      "Train loss: 0.00323444\n",
      "Train loss: 0.00334043\n",
      "Train loss: 0.00678358\n",
      "Train loss: 0.00141164\n",
      "Train loss: 0.00191889\n",
      "Train loss: 0.00426745\n",
      "Train loss: 0.00844681\n",
      "Train loss: 0.00550974\n",
      "Train loss: 0.00263159\n",
      "Train loss: 0.0050319\n",
      "Train loss: 0.00269374\n",
      "Train loss: 0.00456904\n",
      "Train loss: 0.00436429\n",
      "Train loss: 0.00233532\n",
      "Train loss: 0.00506232\n",
      "Train loss: 0.00298044\n",
      "Train loss: 0.00369083\n",
      "Train loss: 0.00413834\n",
      "Train loss: 0.00469195\n",
      "Train loss: 0.00640808\n",
      "Train loss: 0.00288437\n",
      "Train loss: 0.0041524\n",
      "Train loss: 0.00529025\n",
      "Train loss: 0.00678229\n",
      "Train loss: 0.00527402\n",
      "Train loss: 0.00346868\n",
      "Train loss: 0.00498287\n",
      "Train loss: 0.00278071\n",
      "Train loss: 0.00187733\n",
      "Train loss: 0.00553654\n",
      "Train loss: 0.00253756\n",
      "Train loss: 0.00277462\n",
      "Train loss: 0.00765796\n",
      "Train loss: 0.00876946\n",
      "Train loss: 0.00573689\n",
      "Train loss: 0.00317596\n",
      "Train loss: 0.00271139\n",
      "Train loss: 0.0042992\n",
      "Train loss: 0.00310439\n",
      "Train loss: 0.00600758\n",
      "Train loss: 0.00162966\n",
      "Train loss: 0.00490991\n",
      "Train loss: 0.00284936\n",
      "Train loss: 0.00278657\n",
      "Train loss: 0.00545869\n",
      "Train loss: 0.00317017\n",
      "Train loss: 0.00443295\n",
      "Train loss: 0.00198986\n",
      "Train loss: 0.00378864\n",
      "Train loss: 0.00359344\n",
      "Train loss: 0.00211329\n",
      "Train loss: 0.0032264\n",
      "Train loss: 0.00469634\n",
      "Train loss: 0.00316607\n",
      "Train loss: 0.00439523\n",
      "Train loss: 0.00244899\n",
      "Train loss: 0.0041591\n",
      "Train loss: 0.00225802\n",
      "Train loss: 0.00653663\n",
      "Train loss: 0.00424031\n",
      "Train loss: 0.00442796\n",
      "Train loss: 0.00329968\n",
      "Train loss: 0.00337095\n",
      "Train loss: 0.00228221\n",
      "Train loss: 0.00419669\n",
      "Train loss: 0.00341429\n",
      "Train loss: 0.00545543\n",
      "Train loss: 0.00558925\n",
      "Train loss: 0.00599158\n",
      "Train loss: 0.00262276\n",
      "Train loss: 0.00577357\n",
      "Train loss: 0.00189631\n",
      "Train loss: 0.00575262\n",
      "Train loss: 0.00645137\n",
      "Train loss: 0.00377125\n",
      "Train loss: 0.00417441\n",
      "Train loss: 0.00224612\n",
      "Train loss: 0.00599109\n",
      "Train loss: 0.00555201\n",
      "Train loss: 0.00278817\n",
      "Train loss: 0.00145859\n",
      "Train loss: 0.00331832\n",
      "Train loss: 0.00154156\n",
      "Train loss: 0.00129112\n",
      "Train loss: 0.00356186\n",
      "Train loss: 0.00291947\n",
      "Train loss: 0.00342227\n",
      "Train loss: 0.00286931\n",
      "Train loss: 0.00609352\n",
      "Train loss: 0.00218576\n",
      "Train loss: 0.00563683\n",
      "Train loss: 0.00637965\n",
      "Train loss: 0.00296135\n",
      "Train loss: 0.00168554\n",
      "Train loss: 0.0019721\n",
      "Train loss: 0.00266583\n",
      "Train loss: 0.00396409\n",
      "Train loss: 0.00180536\n",
      "Train loss: 0.00400654\n",
      "Train loss: 0.0056218\n",
      "Train loss: 0.00440602\n",
      "Train loss: 0.00240723\n",
      "Train loss: 0.00499984\n",
      "Train loss: 0.0020682\n",
      "Train loss: 0.00142396\n",
      "Train loss: 0.00243187\n",
      "Train loss: 0.00272901\n",
      "Train loss: 0.00304142\n",
      "Train loss: 0.0043561\n",
      "Train loss: 0.00348304\n",
      "Train loss: 0.00369817\n",
      "Train loss: 0.00384144\n",
      "Train loss: 0.00185333\n",
      "Train loss: 0.00532972\n",
      "Train loss: 0.00438807\n",
      "Train loss: 0.00547867\n",
      "Train loss: 0.00216647\n",
      "Train loss: 0.00191177\n",
      "Train loss: 0.00250293\n",
      "Train loss: 0.00272275\n",
      "Train loss: 0.0067725\n",
      "Train loss: 0.00172666\n",
      "Train loss: 0.00458065\n",
      "Train loss: 0.00231477\n",
      "Train loss: 0.0041212\n",
      "Train loss: 0.00172493\n",
      "Train loss: 0.00413154\n",
      "Train loss: 0.00238614\n",
      "Train loss: 0.00472832\n",
      "Train loss: 0.00221589\n",
      "Train loss: 0.00172738\n",
      "Train loss: 0.00370991\n",
      "Train loss: 0.00325974\n",
      "Train loss: 0.0046034\n",
      "Train loss: 0.00176644\n",
      "Train loss: 0.00344497\n",
      "Train loss: 0.00255293\n",
      "Train loss: 0.00121847\n",
      "Train loss: 0.0095855\n",
      "Train loss: 0.00607232\n",
      "Train loss: 0.00183931\n",
      "Train loss: 0.00213725\n",
      "Train loss: 0.00656994\n",
      "Train loss: 0.00308793\n",
      "Train loss: 0.00183036\n",
      "Train loss: 0.00243418\n",
      "Train loss: 0.00148633\n",
      "Train loss: 0.0074691\n",
      "Train loss: 0.00297627\n",
      "Train loss: 0.00331215\n",
      "Train loss: 0.00355512\n",
      "Train loss: 0.00241373\n",
      "Train loss: 0.00284795\n",
      "Train loss: 0.00322987\n",
      "Train loss: 0.00320745\n",
      "Train loss: 0.00239734\n",
      "Train loss: 0.00303494\n",
      "Train loss: 0.00396657\n",
      "Train loss: 0.00307376\n",
      "Train loss: 0.00182809\n",
      "Train loss: 0.005588\n",
      "Train loss: 0.00351471\n",
      "Train loss: 0.00177573\n",
      "Train loss: 0.00214489\n",
      "Train loss: 0.00309926\n",
      "Train loss: 0.00280498\n",
      "Train loss: 0.00220939\n",
      "Train loss: 0.0040472\n",
      "Train loss: 0.00392864\n",
      "Train loss: 0.00621727\n",
      "Train loss: 0.00192934\n",
      "Train loss: 0.00225156\n",
      "Train loss: 0.00400149\n",
      "Train loss: 0.00173929\n",
      "Train loss: 0.00209858\n",
      "Train loss: 0.00117028\n",
      "Train loss: 0.00584217\n",
      "Train loss: 0.00942674\n",
      "Train loss: 0.00149529\n",
      "Train loss: 0.00148022\n",
      "Train loss: 0.00127838\n",
      "Train loss: 0.00235198\n",
      "Train loss: 0.00497641\n",
      "Train loss: 0.00156623\n",
      "Train loss: 0.00558592\n",
      "Train loss: 0.00292031\n",
      "Train loss: 0.00139054\n",
      "Train loss: 0.00349574\n",
      "Train loss: 0.00298053\n",
      "Train loss: 0.0025122\n",
      "Train loss: 0.00165117\n",
      "Train loss: 0.00567081\n",
      "Train loss: 0.00130438\n",
      "Train loss: 0.00106052\n",
      "Train loss: 0.0026855\n",
      "Train loss: 0.00292729\n",
      "Train loss: 0.00541993\n",
      "Train loss: 0.00241096\n",
      "Train loss: 0.00297492\n",
      "Train loss: 0.00404649\n",
      "Train loss: 0.0029581\n",
      "Train loss: 0.00239177\n"
     ]
    }
   ],
   "source": [
    "with sess.as_default():\n",
    "    sess.run([init_vars_op, init_tables_op])\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "    for i in range(TRAIN_ITERATIONS):\n",
    "        train_loss,_ = sess.run([loss_op, train_op])\n",
    "        print \"Train loss: %s\" % train_loss\n",
    "    coord.request_stop()\n",
    "    coord.join(threads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving model using tf.saved_model so we can serve easily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /tmp/tf_tutorial/model/0/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.saved_model import builder as saved_model_builder\n",
    "from tensorflow.python.saved_model import signature_constants\n",
    "from tensorflow.python.saved_model import tag_constants\n",
    "from tensorflow.python.saved_model import signature_def_utils\n",
    "from tensorflow.python.saved_model import utils\n",
    "\n",
    "! rm -rf /tmp/tf_tutorial/model/0\n",
    "\n",
    "with sess.as_default():    \n",
    "    legacy_init_op = tf.group(\n",
    "          tf.tables_initializer(), name='legacy_init_op')\n",
    "    \n",
    "    builder = saved_model_builder.SavedModelBuilder('/tmp/tf_tutorial/model/0')\n",
    "    \n",
    "    signature = signature_def_utils.build_signature_def(\n",
    "          inputs={'content': utils.build_tensor_info(content_batch_op)},\n",
    "          outputs={'logits': utils.build_tensor_info(logits),\n",
    "                   'prediction': utils.build_tensor_info(prediction)},\n",
    "          method_name=signature_constants.PREDICT_METHOD_NAME)    \n",
    "    \n",
    "    builder.add_meta_graph_and_variables(\n",
    "                        sess, \n",
    "                        [tag_constants.SERVING],\n",
    "                        signature_def_map={\n",
    "                            'predict_language': signature,\n",
    "                        },\n",
    "                        main_op=legacy_init_op\n",
    "                        )                \n",
    "    builder.save()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}