seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import math\n",
    "import numpy as np\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "import pandas as pd\n",
    "import itertools\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from functools import partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# bible\n",
    "def sparse(x):\n",
    "    t = [0] * vocab_size\n",
    "    t[x] = 1\n",
    "    return t\n",
    "def process(x):\n",
    "    t = [sparse(w_i[i]) for i in x.split()]\n",
    "    return t\n",
    "\n",
    "pairs = np.load('pairs.npy')\n",
    "w_i = np.load('word2index.npy').item()\n",
    "i_w = np.load('index2word.npy')\n",
    "vocab_size = len(i_w)\n",
    "pairs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# conv.txt\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "def process_sentence(x, eos=False):\n",
    "#     x = x.split()\n",
    "    x = tokenizer.tokenize(x)\n",
    "    x = [i.lower() for i in x]\n",
    "    if eos:\n",
    "        x = ['eos'] + x + ['eos']\n",
    "    return x\n",
    "\n",
    "def one_hot(x):\n",
    "    t = [0] * vocab_size\n",
    "    t[x] = 1\n",
    "    return t\n",
    "    \n",
    "def sent_encode(x):\n",
    "    x = [w_i[i] for i in x]\n",
    "    x = map(one_hot, x)\n",
    "    return list(x)\n",
    "\n",
    "data=pd.read_csv('conv.txt', delimiter='\\t', error_bad_lines=False, header=None)\n",
    "data.dropna(inplace=True)\n",
    "inp = list(map(partial(process_sentence), data[0].values))\n",
    "out = list(map(partial(process_sentence, eos=True), data[1].values))\n",
    "words=np.r_[inp, out]\n",
    "words = list(set(itertools.chain(*words)))\n",
    "vocab_size=len(words)\n",
    "i_w = dict(enumerate(words))\n",
    "w_i = {v:k for k, v in i_w.items()}\n",
    "inp = list(map(sent_encode, inp))\n",
    "out = list(map(sent_encode, out))\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_size = vocab_size\n",
    "print(input_size)\n",
    "dims = [300, 200, 300]\n",
    "num_layers = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def recurrence(prev, inp):\n",
    "    i = tf.reshape(inp, shape=[1, -1])\n",
    "    p = tf.reshape(prev, shape=[1, -1])\n",
    "    z = tf.nn.sigmoid(tf.matmul(i, enc_layers[j]['Wxz']) + tf.matmul(p, enc_layers[j]['Whz']))\n",
    "    r = tf.nn.sigmoid(tf.matmul(i, enc_layers[j]['Wxr']) + tf.matmul(p, enc_layers[j]['Whr']))\n",
    "    h_ = tf.nn.tanh(tf.matmul(i, enc_layers[j]['Wxh']) +\n",
    "                    tf.matmul(tf.mul(p, r), enc_layers[j]['Whh']))\n",
    "    h = tf.mul(tf.sub(tf.ones_like(z), z), h_) + tf.mul(z, p)\n",
    "    return tf.reshape(h, [dims[j-1]])\n",
    "\n",
    "def recurrence2(prev, inp):\n",
    "    i = tf.reshape(inp, shape=[1, -1])\n",
    "    p = tf.reshape(prev, shape=[1, -1])\n",
    "    z = tf.nn.sigmoid(tf.matmul(i, dec_layers[j]['Wxz']) + tf.matmul(p, dec_layers[j]['Whz']))\n",
    "    r = tf.nn.sigmoid(tf.matmul(i, dec_layers[j]['Wxr']) + tf.matmul(p, dec_layers[j]['Whr']))\n",
    "    h_ = tf.nn.tanh(tf.matmul(i, dec_layers[j]['Wxh']) +\n",
    "                    tf.matmul(tf.mul(p, r), dec_layers[j]['Whh']))\n",
    "    h = tf.mul(tf.sub(tf.ones_like(z), z), h_) + tf.mul(z, p)\n",
    "    return tf.reshape(h, [dims[j-1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "with tf.device('/cpu:0'):\n",
    "    with tf.name_scope('encoder'):\n",
    "        enc_layers = []\n",
    "        incoming = tf.placeholder(shape=[None, input_size], dtype=tf.float32, name='encoder_incoming')\n",
    "        enc_layers.append({'states': incoming, 'dim': input_size})\n",
    "        \n",
    "        for j in range(1, num_layers + 1):\n",
    "            shape = [enc_layers[-1]['dim'], dims[j-1]]\n",
    "            layer = dict()\n",
    "            layer['Wxh'] = tf.Variable(tf.random_normal(\n",
    "                shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whh'] = tf.Variable(tf.random_normal(\n",
    "                [dims[j-1], dims[j-1]], stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "            \n",
    "        # layer associated with update gate\n",
    "            layer['Wxz'] = tf.Variable(tf.random_normal(shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whz'] = tf.Variable(tf.random_normal(shape=[\n",
    "                dims[j-1], dims[j-1]], stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "        # layer associated with the reset gate\n",
    "            layer['Wxr'] = tf.Variable(tf.random_normal(shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whr'] = tf.Variable(tf.random_normal(shape=[\n",
    "                dims[j-1], dims[j-1]],\n",
    "                stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "            layer['dim'] = dims[j-1]\n",
    "            layer['initial'] = tf.placeholder(shape=[dims[j-1]], dtype=tf.float32, name='initial'+str(j))\n",
    "            enc_layers.append(layer)\n",
    "            layer['states'] = tf.scan(recurrence, enc_layers[-2]['states'], initializer=layer['initial'], name='encoder_states'+str(j))\n",
    "            layer['last'] = tf.squeeze(tf.expand_dims(tf.nn.embedding_lookup(layer['states'], \n",
    "                                                                             tf.shape(layer['states'])[0] - 1), dim=0),\n",
    "                                      name='last'+str(j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with tf.device('/cpu:0'):\n",
    "    with tf.name_scope('decoder'):\n",
    "        \n",
    "        dec_layers = []\n",
    "        decoder_input = tf.placeholder(shape=[None, input_size], dtype=tf.float32, name='decoeder_input')\n",
    "        dec_layers.append({'states': decoder_input, 'dim': input_size})\n",
    "        \n",
    "        for j in range(1, num_layers + 1):\n",
    "            shape = [dec_layers[-1]['dim'], dims[j-1]]\n",
    "            layer = dict()\n",
    "            layer['Wxh'] = tf.Variable(tf.random_normal(\n",
    "                shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whh'] = tf.Variable(tf.random_normal(\n",
    "                [dims[j-1], dims[j-1]], stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "            \n",
    "        # layer associated with update gate\n",
    "            layer['Wxz'] = tf.Variable(tf.random_normal(shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whz'] = tf.Variable(tf.random_normal(shape=[\n",
    "                dims[j-1], dims[j-1]], stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "        # layer associated with the reset gate\n",
    "            layer['Wxr'] = tf.Variable(tf.random_normal(shape,\n",
    "                stddev=1.0 / math.sqrt(shape[0])), dtype=tf.float32)\n",
    "            layer['Whr'] = tf.Variable(tf.random_normal(shape=[\n",
    "                dims[j-1], dims[j-1]],\n",
    "                stddev=1.0 / math.sqrt(dims[j-1])), dtype=tf.float32)\n",
    "            layer['dim'] = dims[j-1]\n",
    "            dec_layers.append(layer)\n",
    "            \n",
    "            layer['states'] = tf.scan(recurrence2, dec_layers[-2]['states'], initializer=enc_layers[j]['last'], \n",
    "                                      name='decoder_states'+str(j))\n",
    "            \n",
    "\n",
    "        # softmax layer\n",
    "        Why = tf.Variable(tf.random_normal(\n",
    "            [dims[-1], input_size],\n",
    "            stddev=1.0 / math.sqrt(dims[-1])), dtype=tf.float32)\n",
    "        by = tf.Variable(tf.zeros(input_size), dtype=tf.float32)\n",
    "        outputs = tf.matmul(dec_layers[-1]['states'], Why) + by\n",
    "        \n",
    "        # decoding step (with 'teacher forcing')\n",
    "\n",
    "        decoder_target = tf.placeholder(shape=[None, input_size], dtype=tf.float32, name='decoder_target')\n",
    "        \n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(outputs, decoder_target))\n",
    "        optimizer_op = tf.train.AdagradOptimizer(0.1).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()\n",
    "sess =  tf.Session()\n",
    "saver.restore(sess, 'model.ckpt')\n",
    "# sess.run(tf.initialize_all_variables())\n",
    "ix = 0\n",
    "avg_loss = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# beam search output generator (explained here: http://goo.gl/g3jjYs)\n",
    "\n",
    "def beam_generate(inp, size=2):\n",
    "    inp = process_sentence(inp)\n",
    "    inp = sent_encode(inp)\n",
    "    out = ['eos']\n",
    "    seed = [   [one_hot(w_i['eos'])]   ]\n",
    "    temp = []\n",
    "    for j in range(10):\n",
    "        for s in seed:\n",
    "            feed = {k:v for k,v in zip([l['initial'] for l in enc_layers[1:]], [np.zeros(d) for d in dims])}\n",
    "            feed.update({enc_layers[0]['states'] : inp, dec_layers[0]['states']: s})        \n",
    "            o = sess.run(outputs, feed_dict=feed)[-1]\n",
    "            r = np.argsort(o)[-size:]\n",
    "            t = list(np.argmax(s, axis=1))\n",
    "#             print('r', r)\n",
    "#             print('t', t)\n",
    "            for i in r:\n",
    "#                 print('t+i', t + [i])\n",
    "                temp.append(s + [one_hot(i)])\n",
    "#         print(len(temp))\n",
    "        temp = temp[:size]\n",
    "        seed = temp\n",
    "        temp = []\n",
    "    seed=[np.argmax(s, axis=1) for s in seed]\n",
    "    for s in seed:\n",
    "        t=''\n",
    "        for j in s:\n",
    "            t+=' '+i_w[j]\n",
    "        print(t)\n",
    "        \n",
    "            \n",
    "def generate_out(inp):       \n",
    "    inp = process_sentence(inp)\n",
    "    inp = sent_encode(inp)\n",
    "    out = ['eos']\n",
    "    seed = [one_hot(w_i['eos'])]\n",
    "    while len(seed) < 20:\n",
    "        feed = {k:v for k,v in zip([l['initial'] for l in enc_layers[1:]], [np.zeros(d) for d in dims])}\n",
    "        feed.update({enc_layers[0]['states'] : inp, dec_layers[0]['states']: seed})\n",
    "        o = sess.run(outputs, feed_dict=feed)[-1]\n",
    "        r = i_w[np.argmax(o)]\n",
    "        seed.append(o)\n",
    "        out.append(r)\n",
    "        if r == 'eos':\n",
    "            break\n",
    "    print(' '.join(out[1:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print_step = 1000\n",
    "for i in range(2 * len(inp)):\n",
    "    ix = np.random.randint(0, len(inp))\n",
    "    a = inp[ix]\n",
    "    b = out[ix]\n",
    "    feed = {incoming: a, decoder_target: b[1:], decoder_input : b[:-1]}\n",
    "    for j in enc_layers[1:]:\n",
    "        feed[j['initial']] = np.zeros(j['dim'])\n",
    "        \n",
    "    l, _, o = sess.run([loss, optimizer_op, outputs], feed_dict=feed)\n",
    "    avg_loss += l\n",
    "    if i % print_step == 0 and i != 0:\n",
    "        print(avg_loss / print_step)\n",
    "        avg_loss = 0\n",
    "        saver.save(sess, 'checkpoint.ckpt')\n",
    "        generate_out('do you have girlfriend')\n",
    "        \n",
    "    ix += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# saver = tf.train.Saver()\n",
    "saver.save(sess, 'model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "generate_out('thanks')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "[i_w[x] for x in np.argmax(inp[0], axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}