notebooks/char_rnn_sample_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages Imported\n"
     ]
    }
   ],
   "source": [
    "# Import Packages\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "import argparse\n",
    "import time\n",
    "import os\n",
    "from six.moves import cPickle\n",
    "print (\"Packages Imported\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'vocab_size' is 99\n"
     ]
    }
   ],
   "source": [
    "# Load chars and vocab\n",
    "load_dir = \"data/linux_kernel\"\n",
    "with open(os.path.join(load_dir, 'chars_vocab.pkl'), 'rb') as f:\n",
    "    chars, vocab = cPickle.load(f)\n",
    "vocab_size = len(vocab) \n",
    "print (\"'vocab_size' is %d\" % (vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now, we are ready to make our RNN model with seq2seq\n",
    "### This network is for sampling, so we don't need batches for sequenes nor optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready\n"
     ]
    }
   ],
   "source": [
    "# Important RNN parameters \n",
    "rnn_size   = 128\n",
    "num_layers = 2\n",
    "\n",
    "batch_size = 1 # <= In the training phase, these were both 50\n",
    "seq_length = 1\n",
    "\n",
    "# Construct RNN model \n",
    "unitcell   = tf.nn.rnn_cell.BasicLSTMCell(rnn_size)\n",
    "cell       = tf.nn.rnn_cell.MultiRNNCell([unitcell] * num_layers)\n",
    "input_data = tf.placeholder(tf.int32, [batch_size, seq_length])\n",
    "istate     = cell.zero_state(batch_size, tf.float32)\n",
    "# Weigths \n",
    "with tf.variable_scope('rnnlm'):\n",
    "    softmax_w = tf.get_variable(\"softmax_w\", [rnn_size, vocab_size])\n",
    "    softmax_b = tf.get_variable(\"softmax_b\", [vocab_size])\n",
    "    with tf.device(\"/cpu:0\"):\n",
    "        embedding = tf.get_variable(\"embedding\", [vocab_size, rnn_size])\n",
    "        inputs = tf.split(1, seq_length, tf.nn.embedding_lookup(embedding, input_data))\n",
    "        inputs = [tf.squeeze(_input, [1]) for _input in inputs]\n",
    "# Output\n",
    "def loop(prev, _):\n",
    "    prev = tf.nn.xw_plus_b(prev, softmax_w, softmax_b)\n",
    "    prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))\n",
    "    return tf.nn.embedding_lookup(embedding, prev_symbol)\n",
    "outputs, final_state = seq2seq.rnn_decoder(inputs, istate, cell\n",
    "                                          , loop_function=None, scope='rnnlm')\n",
    "output = tf.reshape(tf.concat(1, outputs), [-1, rnn_size])\n",
    "\n",
    "logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)\n",
    "probs  = tf.nn.softmax(logits)\n",
    "\n",
    "print (\"Network Ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/linux_kernel/model.ckpt-8000\n"
     ]
    }
   ],
   "source": [
    "# Restore RNN\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "saver = tf.train.Saver(tf.all_variables())\n",
    "ckpt  = tf.train.get_checkpoint_state(load_dir)\n",
    "\n",
    "print (ckpt.model_checkpoint_path)\n",
    "saver.restore(sess, ckpt.model_checkpoint_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finally, show what RNN has generated! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling Done. \n",
      "___________________________________________\n",
      "\n",
      "/*  :  A C. Fruemptly etweennars must be serversed */\n",
      "static int __cgroup_hash_power(struct rt_mutex_d *uaddr, int watab, long\n",
      "-XIT_PYS__AUTIMER_PAT(seed_class_table_watch, v1->curr);\n",
      "}\n",
      "\n",
      "static void down_cpusets(struct pid;\n",
      "static int pid_thread(voids_mm)\n",
      "{\n",
      "\tif (ps->cpumainte_to_cgroup_grp <= NULL)\n",
      "\t\treturn 0;\n",
      "}\n",
      "\n",
      "conset sched_VRICE_SOFTIRQ_DISU{\n",
      "\tsoftirq_signal(this_css_set_bytes));\n",
      "}\n",
      "\n",
      "void private = {\n",
      "\t\t.mode\t\t= CPUCLOCK_BALANCE,\n",
      "\t\t.process\t\t= optime)\n",
      "\n",
      "/*\n",
      " * The are\n",
      " *\ten\n",
      " * @buf' - for so allows the condext it of it regions)\n",
      " * massessiging that   Sto be stime in the expoxes\n",
      " */\n",
      "void __fsix;\n",
      "\tstruct audit_chunk *tsk;\n",
      "\n",
      "\tkey_utvec_oper(struct *read_ns, struct futex_ckernel);\n",
      "\tint atomic_attime = res->init_switch(void),\n",
      "\t\t\t    -+signal->state = 0;\n",
      "\ttmr = tmp;\n",
      "\tprintk(\"%s\\n\", signal, &max_huts_string, 1, look_t *)(modemask++);\n",
      "\tup_sem(cft, &(max))) {\n",
      "\t\tif (probes)\n",
      "\t\t\tset_cpu(name == 0)\n",
      "\t\t\tgoto out;\n",
      "\t}\n",
      "\n",
      "\tpposs_unlock(*pefmask_plocks);\n",
      "\taudit_log_lock_fuces(rq);\n",
      "}\n",
      "\n",
      "static void again;\n",
      "\n",
      "int\n",
      "con\n"
     ]
    }
   ],
   "source": [
    "# Sampling function\n",
    "def weighted_pick(weights):\n",
    "    t = np.cumsum(weights)\n",
    "    s = np.sum(weights)\n",
    "    return(int(np.searchsorted(t, np.random.rand(1)*s)))\n",
    "\n",
    "# Sample using RNN and prime characters\n",
    "prime = \"/* \"\n",
    "state = sess.run(cell.zero_state(1, tf.float32))\n",
    "for char in prime[:-1]:\n",
    "    x = np.zeros((1, 1))\n",
    "    x[0, 0] = vocab[char]\n",
    "    state = sess.run(final_state, feed_dict={input_data: x, istate:state})\n",
    "\n",
    "# Sample 'num' characters\n",
    "ret  = prime\n",
    "char = prime[-1] # <= This goes IN! \n",
    "num  = 1000\n",
    "for n in range(num):\n",
    "    x = np.zeros((1, 1))\n",
    "    x[0, 0] = vocab[char]\n",
    "    [probsval, state] = sess.run([probs, final_state]\n",
    "        , feed_dict={input_data: x, istate:state})\n",
    "    p      = probsval[0] \n",
    "    \n",
    "    sample = weighted_pick(p)\n",
    "    # sample = np.argmax(p)\n",
    "    \n",
    "    pred   = chars[sample]\n",
    "    ret    = ret + pred\n",
    "    char   = pred\n",
    "    \n",
    "print (\"Sampling Done. \\n___________________________________________\\n\")\n",
    "\n",
    "print (ret)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hope, it was good. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}