lstm_main_journey.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "rnn_cell = tf.nn.rnn_cell\n",
    "\n",
    "sequence_loss_by_example = tf.nn.seq2seq.sequence_loss_by_example\n",
    "# parses the dataset\n",
    "import ptb_reader\n",
    "\n",
    "# import variants\n",
    "from variants.vanilla import VanillaLSTMCell\n",
    "# from variants.nig import NIGLSTMCell\n",
    "# from variants.nfg import NFGLSTMCell\n",
    "# from variants.nog import NOGLSTMCell\n",
    "# from variants.niaf import NIAFLSTMCell\n",
    "# from variants.noaf import NOAFLSTMCell\n",
    "# from variants.np import NPLSTMCell\n",
    "# from variants.cifg import CIFGLSTMCell\n",
    "# from variants.fgr import FGRLSTMCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = '../../Machine175/AttentionModels/rnn/data/simple-examples/data/'\n",
    "train_data, valid_data, test_data, _ = ptb_reader.ptb_raw_data(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define artifact directories where results from the session can be saved\n",
    "model_path = os.environ.get('MODEL_PATH', 'models/')\n",
    "checkpoint_path = os.environ.get('CHECKPOINT_PATH', 'checkpoints/')\n",
    "summary_path = os.environ.get('SUMMARY_PATH', 'logs/')\n",
    "\n",
    "\n",
    "def write_csv(arr, path):\n",
    "    df = pd.DataFrame(arr)\n",
    "    df.to_csv(path)\n",
    "\n",
    "class PTBModel(object):\n",
    "    def __init__(self, CellType, is_training, config):\n",
    "        self.batch_size = batch_size = config.batch_size\n",
    "        self.num_steps = num_steps = config.num_steps\n",
    "        size = config.hidden_size\n",
    "        vocab_size = config.vocab_size\n",
    "\n",
    "        self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps], name=\"input_data\")\n",
    "        self.targets = tf.placeholder(tf.int32, [batch_size, num_steps], name=\"targets\")\n",
    "\n",
    "        lstm_cell = CellType(size)\n",
    "        if is_training and config.keep_prob < 1:\n",
    "            lstm_cell = rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=config.keep_prob)\n",
    "        cell = rnn_cell.MultiRNNCell([lstm_cell] * config.num_layers)\n",
    "        self.initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "\n",
    "        # initializer used for reusable variable initializer (see `get_variable`)\n",
    "        initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale)\n",
    "\n",
    "        with tf.device(\"/cpu:0\"):\n",
    "            embedding = tf.get_variable(\"embedding\", [vocab_size, size], initializer=initializer)\n",
    "            inputs = tf.nn.embedding_lookup(embedding, self.input_data)\n",
    "\n",
    "        if is_training and config.keep_prob < 1:\n",
    "            inputs = tf.nn.dropout(inputs, config.keep_prob)\n",
    "\n",
    "        outputs = []\n",
    "        states = []\n",
    "        state = self.initial_state\n",
    "\n",
    "        with tf.variable_scope(\"RNN\", initializer=initializer):\n",
    "            for time_step in range(num_steps):\n",
    "                if time_step > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                inputs_slice = inputs[:,time_step,:]\n",
    "                (cell_output, state) = cell(inputs_slice, state)\n",
    "\n",
    "                outputs.append(cell_output)\n",
    "                states.append(state)\n",
    "\n",
    "        self.final_state = states[-1]\n",
    "\n",
    "        output = tf.reshape(tf.concat(1, outputs), [-1, size])\n",
    "        w = tf.get_variable(\"softmax_w\",\n",
    "                                    [size, vocab_size],\n",
    "                                    initializer=initializer)\n",
    "        b = tf.get_variable(\"softmax_b\", [vocab_size], initializer=initializer)\n",
    "\n",
    "        logits = tf.nn.xw_plus_b(output, w, b) # compute logits for loss\n",
    "        targets = tf.reshape(self.targets, [-1]) # reshape our target outputs\n",
    "        weights = tf.ones([batch_size * num_steps]) # used to scale the loss average\n",
    "\n",
    "        # computes loss and performs softmax on our fully-connected output layer\n",
    "        loss = sequence_loss_by_example([logits], [targets], [weights], vocab_size)\n",
    "        self.cost = cost = tf.div(tf.reduce_sum(loss), batch_size, name=\"cost\")\n",
    "\n",
    "        if is_training:\n",
    "            # setup learning rate variable to decay\n",
    "            self.lr = tf.Variable(1.0, trainable=False)\n",
    "\n",
    "            # define training operation and clip the gradients\n",
    "            tvars = tf.trainable_variables()\n",
    "            grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), config.max_grad_norm)\n",
    "            optimizer = tf.train.GradientDescentOptimizer(self.lr)\n",
    "            self.train_op = optimizer.apply_gradients(zip(grads, tvars), name=\"train\")\n",
    "        else:\n",
    "            # if this model isn't for training (i.e. testing/validation) then we don't do anything here\n",
    "            self.train_op = tf.no_op()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Learning Rate: 1.000\n",
      "0.0% Perplexity: 9980.425 (Cost: 322.293) Speed: 431 wps\n",
      "0.8% Perplexity: 7468.040 (Cost: 279.900) Speed: 2428 wps\n",
      "1.5% Perplexity: 4879.515 (Cost: 258.628) Speed: 3162 wps\n",
      "2.3% Perplexity: 3627.752 (Cost: 254.055) Speed: 3540 wps\n",
      "3.0% Perplexity: 2899.267 (Cost: 251.960) Speed: 3775 wps\n",
      "3.8% Perplexity: 2456.090 (Cost: 239.738) Speed: 3937 wps\n",
      "4.5% Perplexity: 2231.918 (Cost: 242.574) Speed: 4048 wps\n",
      "5.3% Perplexity: 1995.805 (Cost: 240.357) Speed: 4132 wps\n",
      "6.0% Perplexity: 1829.456 (Cost: 240.041) Speed: 4202 wps\n",
      "6.8% Perplexity: 1686.089 (Cost: 238.162) Speed: 4258 wps\n",
      "7.5% Perplexity: 1571.471 (Cost: 229.152) Speed: 4305 wps\n",
      "8.3% Perplexity: 1489.269 (Cost: 234.274) Speed: 4340 wps\n",
      "9.0% Perplexity: 1410.841 (Cost: 232.232) Speed: 4371 wps\n",
      "9.8% Perplexity: 1334.884 (Cost: 228.339) Speed: 4399 wps\n",
      "10.6% Perplexity: 1278.505 (Cost: 225.641) Speed: 4423 wps\n",
      "11.3% Perplexity: 1227.590 (Cost: 230.127) Speed: 4442 wps\n",
      "12.1% Perplexity: 1179.076 (Cost: 216.604) Speed: 4459 wps\n",
      "12.8% Perplexity: 1146.219 (Cost: 230.370) Speed: 4475 wps\n",
      "13.6% Perplexity: 1110.807 (Cost: 228.654) Speed: 4489 wps\n",
      "14.3% Perplexity: 1074.283 (Cost: 220.122) Speed: 4502 wps\n",
      "15.1% Perplexity: 1041.609 (Cost: 220.708) Speed: 4512 wps\n",
      "15.8% Perplexity: 1013.542 (Cost: 217.105) Speed: 4523 wps\n",
      "16.6% Perplexity: 985.157 (Cost: 223.989) Speed: 4533 wps\n",
      "17.3% Perplexity: 960.458 (Cost: 215.321) Speed: 4542 wps\n",
      "18.1% Perplexity: 934.190 (Cost: 216.513) Speed: 4550 wps\n",
      "18.8% Perplexity: 913.557 (Cost: 214.539) Speed: 4557 wps\n",
      "19.6% Perplexity: 893.968 (Cost: 217.643) Speed: 4565 wps\n",
      "20.3% Perplexity: 878.025 (Cost: 223.081) Speed: 4572 wps\n",
      "21.1% Perplexity: 859.768 (Cost: 208.841) Speed: 4577 wps\n",
      "21.9% Perplexity: 842.868 (Cost: 216.987) Speed: 4582 wps\n",
      "22.6% Perplexity: 823.457 (Cost: 206.566) Speed: 4587 wps\n",
      "23.4% Perplexity: 805.231 (Cost: 211.573) Speed: 4592 wps\n",
      "24.1% Perplexity: 786.171 (Cost: 195.769) Speed: 4597 wps\n",
      "24.9% Perplexity: 771.461 (Cost: 214.771) Speed: 4601 wps\n",
      "25.6% Perplexity: 760.984 (Cost: 223.603) Speed: 4604 wps\n",
      "26.4% Perplexity: 753.383 (Cost: 210.334) Speed: 4608 wps\n",
      "27.1% Perplexity: 741.245 (Cost: 210.033) Speed: 4612 wps\n",
      "27.9% Perplexity: 727.922 (Cost: 223.777) Speed: 4615 wps\n",
      "28.6% Perplexity: 716.694 (Cost: 197.610) Speed: 4618 wps\n",
      "29.4% Perplexity: 706.612 (Cost: 214.410) Speed: 4622 wps\n",
      "30.1% Perplexity: 696.793 (Cost: 215.429) Speed: 4625 wps\n",
      "30.9% Perplexity: 686.051 (Cost: 208.591) Speed: 4628 wps\n",
      "31.7% Perplexity: 677.756 (Cost: 213.857) Speed: 4631 wps\n",
      "32.4% Perplexity: 668.768 (Cost: 202.987) Speed: 4632 wps\n",
      "33.2% Perplexity: 660.098 (Cost: 201.516) Speed: 4635 wps\n",
      "33.9% Perplexity: 651.730 (Cost: 203.988) Speed: 4638 wps\n",
      "34.7% Perplexity: 644.081 (Cost: 216.446) Speed: 4639 wps\n",
      "35.4% Perplexity: 636.876 (Cost: 209.094) Speed: 4641 wps\n",
      "36.2% Perplexity: 628.139 (Cost: 208.938) Speed: 4643 wps\n",
      "36.9% Perplexity: 620.186 (Cost: 214.864) Speed: 4645 wps\n",
      "37.7% Perplexity: 614.377 (Cost: 207.547) Speed: 4647 wps\n",
      "38.4% Perplexity: 609.342 (Cost: 202.756) Speed: 4649 wps\n",
      "39.2% Perplexity: 601.267 (Cost: 198.900) Speed: 4651 wps\n",
      "39.9% Perplexity: 594.340 (Cost: 198.860) Speed: 4653 wps\n",
      "40.7% Perplexity: 590.432 (Cost: 211.892) Speed: 4655 wps\n",
      "41.4% Perplexity: 584.366 (Cost: 199.456) Speed: 4656 wps\n",
      "42.2% Perplexity: 578.291 (Cost: 204.366) Speed: 4658 wps\n",
      "43.0% Perplexity: 571.406 (Cost: 197.463) Speed: 4659 wps\n",
      "43.7% Perplexity: 565.475 (Cost: 206.074) Speed: 4660 wps\n",
      "44.5% Perplexity: 561.525 (Cost: 208.903) Speed: 4662 wps\n",
      "45.2% Perplexity: 558.295 (Cost: 209.760) Speed: 4663 wps\n",
      "46.0% Perplexity: 554.436 (Cost: 200.614) Speed: 4664 wps\n",
      "46.7% Perplexity: 550.187 (Cost: 199.891) Speed: 4666 wps\n",
      "47.5% Perplexity: 545.229 (Cost: 207.607) Speed: 4667 wps\n",
      "48.2% Perplexity: 541.058 (Cost: 196.435) Speed: 4668 wps\n",
      "49.0% Perplexity: 536.281 (Cost: 194.244) Speed: 4669 wps\n",
      "49.7% Perplexity: 532.227 (Cost: 198.494) Speed: 4670 wps\n",
      "50.5% Perplexity: 526.219 (Cost: 189.854) Speed: 4671 wps\n",
      "51.2% Perplexity: 521.356 (Cost: 203.802) Speed: 4672 wps\n",
      "52.0% Perplexity: 517.272 (Cost: 201.385) Speed: 4673 wps\n",
      "52.8% Perplexity: 512.522 (Cost: 198.974) Speed: 4674 wps\n",
      "53.5% Perplexity: 508.969 (Cost: 198.620) Speed: 4675 wps\n",
      "54.3% Perplexity: 504.914 (Cost: 191.261) Speed: 4676 wps\n",
      "55.0% Perplexity: 499.803 (Cost: 187.324) Speed: 4678 wps\n",
      "55.8% Perplexity: 495.784 (Cost: 192.519) Speed: 4678 wps\n",
      "56.5% Perplexity: 492.234 (Cost: 199.191) Speed: 4679 wps\n",
      "57.3% Perplexity: 488.004 (Cost: 193.756) Speed: 4680 wps\n",
      "58.0% Perplexity: 484.177 (Cost: 196.799) Speed: 4681 wps\n",
      "58.8% Perplexity: 479.978 (Cost: 184.612) Speed: 4682 wps\n",
      "59.5% Perplexity: 476.676 (Cost: 195.720) Speed: 4683 wps\n",
      "60.3% Perplexity: 473.547 (Cost: 198.280) Speed: 4684 wps\n",
      "61.0% Perplexity: 470.748 (Cost: 196.725) Speed: 4684 wps\n",
      "61.8% Perplexity: 466.826 (Cost: 200.960) Speed: 4685 wps\n",
      "62.5% Perplexity: 463.265 (Cost: 197.816) Speed: 4685 wps\n",
      "63.3% Perplexity: 460.643 (Cost: 198.235) Speed: 4686 wps\n",
      "64.1% Perplexity: 457.706 (Cost: 193.859) Speed: 4686 wps\n",
      "64.8% Perplexity: 454.568 (Cost: 185.723) Speed: 4687 wps\n",
      "65.6% Perplexity: 451.305 (Cost: 179.607) Speed: 4688 wps\n",
      "66.3% Perplexity: 448.054 (Cost: 185.456) Speed: 4688 wps\n",
      "67.1% Perplexity: 445.984 (Cost: 204.391) Speed: 4689 wps\n",
      "67.8% Perplexity: 443.965 (Cost: 198.055) Speed: 4689 wps\n",
      "68.6% Perplexity: 441.361 (Cost: 200.948) Speed: 4690 wps\n",
      "69.3% Perplexity: 439.026 (Cost: 197.496) Speed: 4691 wps\n",
      "70.1% Perplexity: 436.533 (Cost: 208.952) Speed: 4691 wps\n",
      "70.8% Perplexity: 434.442 (Cost: 195.737) Speed: 4691 wps\n",
      "71.6% Perplexity: 432.426 (Cost: 202.511) Speed: 4692 wps\n",
      "72.3% Perplexity: 430.774 (Cost: 194.549) Speed: 4693 wps\n",
      "73.1% Perplexity: 428.563 (Cost: 195.991) Speed: 4693 wps\n",
      "73.9% Perplexity: 426.713 (Cost: 197.050) Speed: 4694 wps\n",
      "74.6% Perplexity: 424.008 (Cost: 199.578) Speed: 4695 wps\n",
      "75.4% Perplexity: 421.333 (Cost: 191.988) Speed: 4695 wps\n",
      "76.1% Perplexity: 419.577 (Cost: 200.166) Speed: 4695 wps\n",
      "76.9% Perplexity: 417.500 (Cost: 190.451) Speed: 4696 wps\n",
      "77.6% Perplexity: 415.760 (Cost: 193.260) Speed: 4696 wps\n",
      "78.4% Perplexity: 413.221 (Cost: 181.885) Speed: 4696 wps\n",
      "79.1% Perplexity: 410.794 (Cost: 190.209) Speed: 4697 wps\n",
      "79.9% Perplexity: 408.702 (Cost: 183.466) Speed: 4697 wps\n",
      "80.6% Perplexity: 406.115 (Cost: 181.772) Speed: 4698 wps\n",
      "81.4% Perplexity: 403.468 (Cost: 189.114) Speed: 4698 wps\n",
      "82.1% Perplexity: 401.057 (Cost: 198.822) Speed: 4699 wps\n",
      "82.9% Perplexity: 399.287 (Cost: 195.579) Speed: 4699 wps\n",
      "83.6% Perplexity: 397.981 (Cost: 197.500) Speed: 4699 wps\n",
      "84.4% Perplexity: 396.526 (Cost: 190.573) Speed: 4700 wps\n",
      "85.2% Perplexity: 394.791 (Cost: 187.379) Speed: 4700 wps\n",
      "85.9% Perplexity: 393.151 (Cost: 197.027) Speed: 4701 wps\n",
      "86.7% Perplexity: 391.157 (Cost: 173.444) Speed: 4701 wps\n",
      "87.4% Perplexity: 389.655 (Cost: 186.874) Speed: 4701 wps\n",
      "88.2% Perplexity: 387.462 (Cost: 193.311) Speed: 4702 wps\n",
      "88.9% Perplexity: 386.009 (Cost: 182.842) Speed: 4702 wps\n",
      "89.7% Perplexity: 383.735 (Cost: 184.746) Speed: 4702 wps\n",
      "90.4% Perplexity: 381.549 (Cost: 186.875) Speed: 4702 wps\n",
      "91.2% Perplexity: 379.261 (Cost: 183.135) Speed: 4703 wps\n",
      "91.9% Perplexity: 377.242 (Cost: 193.007) Speed: 4703 wps\n",
      "92.7% Perplexity: 375.330 (Cost: 188.485) Speed: 4704 wps\n",
      "93.4% Perplexity: 374.103 (Cost: 195.669) Speed: 4704 wps\n",
      "94.2% Perplexity: 372.221 (Cost: 184.851) Speed: 4704 wps\n",
      "95.0% Perplexity: 370.469 (Cost: 185.488) Speed: 4704 wps\n",
      "95.7% Perplexity: 368.783 (Cost: 184.659) Speed: 4705 wps\n",
      "96.5% Perplexity: 367.302 (Cost: 195.456) Speed: 4705 wps\n",
      "97.2% Perplexity: 366.300 (Cost: 201.566) Speed: 4705 wps\n",
      "98.0% Perplexity: 365.408 (Cost: 194.878) Speed: 4705 wps\n",
      "98.7% Perplexity: 364.103 (Cost: 188.339) Speed: 4705 wps\n",
      "99.5% Perplexity: 363.164 (Cost: 196.834) Speed: 4706 wps\n",
      "Epoch: 1 Training Perplexity: 362.622 (Cost: 5.893)\n",
      "Epoch: 1 Validation Perplexity: 210.816 (Cost: 5.351)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "'latest_filename' collides with 'save_path': 'checkpoint' and 'checkpoints/checkpoint'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-952cf912498a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    108\u001b[0m         \u001b[0mvalid_perps\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_perp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m         \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'checkpoint'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m     \u001b[0;31m# run test pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/training/saver.pyc\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state)\u001b[0m\n\u001b[1;32m   1308\u001b[0m         raise ValueError(\n\u001b[1;32m   1309\u001b[0m             \u001b[0;34m\"'latest_filename' collides with 'save_path': '%s' and '%s'\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1310\u001b[0;31m             (latest_filename, save_path))\n\u001b[0m\u001b[1;32m   1311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1312\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIsDirectory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdirname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: 'latest_filename' collides with 'save_path': 'checkpoint' and 'checkpoints/checkpoint'"
     ]
    }
   ],
   "source": [
    "def run_epoch(sess, model, data, verbose=False):\n",
    "    epoch_size = ((len(data) // model.batch_size) - 1) // model.num_steps\n",
    "    start_time = time.time()\n",
    "\n",
    "    # accumulated counts\n",
    "    costs = 0.0\n",
    "    iters = 0\n",
    "\n",
    "    # initial RNN state\n",
    "#     state = model.initial_state.eval()\n",
    "    state = sess.run(model.initial_state)\n",
    "\n",
    "    for step, (x, y) in enumerate(ptb_reader.ptb_iterator(data, model.batch_size, model.num_steps)):\n",
    "        cost, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed_dict={\n",
    "            model.input_data: x,\n",
    "            model.targets: y,\n",
    "            model.initial_state: state\n",
    "        })\n",
    "        costs += cost\n",
    "        iters += model.num_steps\n",
    "\n",
    "        perplexity = np.exp(costs / iters)\n",
    "\n",
    "        if verbose and step % 10 == 0:\n",
    "            progress = (step / epoch_size) * 100\n",
    "            wps = iters * model.batch_size / (time.time() - start_time)\n",
    "            print(\"%.1f%% Perplexity: %.3f (Cost: %.3f) Speed: %.0f wps\" % (progress, perplexity, cost, wps))\n",
    "\n",
    "    return (costs / iters), perplexity\n",
    "\n",
    "class Config(object):\n",
    "    batch_size = 20\n",
    "    num_steps = 35 # number of unrolled time steps\n",
    "    hidden_size = 450 # number of blocks in an LSTM cell\n",
    "    vocab_size = 10000\n",
    "    max_grad_norm = 5 # maximum gradient for clipping\n",
    "    init_scale = 0.05 # scale between -0.1 and 0.1 for all random initialization\n",
    "    keep_prob = 0.5 # dropout probability\n",
    "    num_layers = 2 # number of LSTM layers\n",
    "    learning_rate = 1.0\n",
    "    lr_decay = 0.8\n",
    "    lr_decay_epoch_offset = 6 # don't decay until after the Nth epoch\n",
    "\n",
    "# default settings for training\n",
    "train_config = Config()\n",
    "\n",
    "# our evaluation runs (validation and testing), use a batch size and time step of one\n",
    "eval_config = Config()\n",
    "eval_config.batch_size = 1\n",
    "eval_config.num_steps = 1\n",
    "\n",
    "# number of epochs to perform over the training data\n",
    "num_epochs = 39\n",
    "\n",
    "cell_types = {\n",
    "    'vanilla': VanillaLSTMCell\n",
    "#     'nig': NIGLSTMCell,\n",
    "#     'nfg': NFGLSTMCell,\n",
    "#     'nog': NOGLSTMCell,\n",
    "#     'niaf': NIAFLSTMCell,\n",
    "#     'noaf': NOAFLSTMCell,\n",
    "#     'np': NPLSTMCell,\n",
    "#     'cifg': CIFGLSTMCell,\n",
    "#     'fgr': FGRLSTMCell,\n",
    "}\n",
    "\n",
    "model_name = \"vanilla\"\n",
    "CellType = cell_types[model_name]\n",
    "\n",
    "with tf.Graph().as_default(), tf.Session() as sess:\n",
    "    # define our training model\n",
    "    with tf.variable_scope(\"model\", reuse=None):\n",
    "        train_model = PTBModel(CellType, is_training=True, config=train_config)\n",
    "\n",
    "    # we create a separate model for validation and testing to alter the batch size and time steps\n",
    "    # reuse=True reuses variables from the previously defined `train_model`\n",
    "    with tf.variable_scope(\"model\", reuse=True):\n",
    "        valid_model = PTBModel(CellType, is_training=False, config=train_config)\n",
    "        test_model = PTBModel(CellType, is_training=False, config=eval_config)\n",
    "\n",
    "    # create a saver instance to restore from the checkpoint\n",
    "    saver = tf.train.Saver(max_to_keep=1)\n",
    "\n",
    "    # initialize our variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # save the graph definition as a protobuf file\n",
    "    tf.train.write_graph(sess.graph_def, model_path, '%s.pb'.format(model_name), as_text=False)\n",
    "\n",
    "    train_costs = []\n",
    "    train_perps = []\n",
    "    valid_costs = []\n",
    "    valid_perps = []\n",
    "\n",
    "    for i in range(num_epochs):\n",
    "        print(\"Epoch: %d Learning Rate: %.3f\" % (i + 1, sess.run(train_model.lr)))\n",
    "\n",
    "        # run training pass\n",
    "        train_cost, train_perp = run_epoch(sess, train_model, train_data, verbose=True)\n",
    "        print(\"Epoch: %i Training Perplexity: %.3f (Cost: %.3f)\" % (i + 1, train_perp, train_cost))\n",
    "        train_costs.append(train_cost)\n",
    "        train_perps.append(train_perp)\n",
    "\n",
    "        # run validation pass\n",
    "        valid_cost, valid_perp = run_epoch(sess, valid_model, valid_data)\n",
    "        print(\"Epoch: %i Validation Perplexity: %.3f (Cost: %.3f)\" % (i + 1, valid_perp, valid_cost))\n",
    "        valid_costs.append(valid_cost)\n",
    "        valid_perps.append(valid_perp)\n",
    "\n",
    "        saver.save(sess, checkpoint_path + 'checkpoint')\n",
    "\n",
    "    # run test pass\n",
    "    test_cost, test_perp = run_epoch(sess, test_model, test_data)\n",
    "    print(\"Test Perplexity: %.3f (Cost: %.3f)\" % (test_perp, test_cost))\n",
    "\n",
    "    write_csv(train_costs, os.path.join(summary_path, \"train_costs.csv\"))\n",
    "    write_csv(train_perps, os.path.join(summary_path, \"train_perps.csv\"))\n",
    "    write_csv(valid_costs, os.path.join(summary_path, \"valid_costs.csv\"))\n",
    "    write_csv(valid_perps, os.path.join(summary_path, \"valid_perps.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for step, (x, y) in enumerate(ptb_reader.ptb_iterator(train_data, batch_size=20, num_steps=35)):\n",
    "#     print(\"step is \",step)\n",
    "#     print(\"x is \", x)\n",
    "#     print(\"y is \", y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "it = enumerate(ptb_reader.ptb_iterator(train_data, batch_size=20, num_steps=35))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "step, (x, y) = next(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 35) (20, 35)\n"
     ]
    }
   ],
   "source": [
    "print (x.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([9979, 9999, 9985, 9976, 9989, 9973, 9983, 9975, 9991, 9970, 9971,\n",
       "        9987, 9995, 9977, 9981, 9972, 9982, 9988, 9974, 9998, 9992, 9993,\n",
       "        9996, 9984,    2, 8998,    1,    3,   72,  393,   33, 2148,    0,\n",
       "         146,   19], dtype=int32),\n",
       " array([9999, 9985, 9976, 9989, 9973, 9983, 9975, 9991, 9970, 9971, 9987,\n",
       "        9995, 9977, 9981, 9972, 9982, 9988, 9974, 9998, 9992, 9993, 9996,\n",
       "        9984,    2, 8998,    1,    3,   72,  393,   33, 2148,    0,  146,\n",
       "          19,    6], dtype=int32))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(x,y)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}