Chapter 9 Encode Decoder LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import seed\n",
    "from random import randint\n",
    "from numpy import array\n",
    "from numpy import argmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_sum_pairs(n_examples, n_numbers, largest):\n",
    "    X,y = list(), list()\n",
    "    for i in range(n_examples):\n",
    "        in_pattern=[randint(1,largest) for _ in range(n_numbers)]\n",
    "        out_pattern = sum(in_pattern)\n",
    "        X.append(in_pattern)\n",
    "        y.append(out_pattern)\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3, 10]] [13]\n"
     ]
    }
   ],
   "source": [
    "seed(1)\n",
    "n_samples =1\n",
    "n_numbers = 2\n",
    "largest = 10\n",
    "X,y = random_sum_pairs(n_samples, n_numbers, largest)\n",
    "print(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import ceil\n",
    "from math import log10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_string(X,y,n_numbers,largest):\n",
    "    max_length = n_numbers*ceil(log10(largest+1)) + n_numbers - 1\n",
    "    Xstr = list()\n",
    "    for pattern in X:\n",
    "        strp = '+'.join([str(n) for n in pattern])\n",
    "        strp = ''.join([' ' for _ in range(max_length-len(strp))]) + strp\n",
    "        Xstr.append(strp)\n",
    "    maxlength = ceil(log10(n_numbers*(largest+1)))\n",
    "    ystr = list()\n",
    "    for pattern in y:\n",
    "        strp = str(pattern)\n",
    "        strp = ''.join([' 'for _ in range(maxlength-len(strp))]) + strp\n",
    "        ystr.append(strp)\n",
    "    return Xstr, ystr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed(1)\n",
    "n_samples = 1\n",
    "n_numbers = 2\n",
    "largest = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3, 10]] [13]\n",
      "[' 3+10'] ['13']\n"
     ]
    }
   ],
   "source": [
    "X,y = random_sum_pairs(n_samples, n_numbers, largest)\n",
    "print(X,y)\n",
    "\n",
    "X,y = to_string(X,y,n_numbers,largest)\n",
    "print(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphabet = ['0','1','2','3','4','5','6','7','8','9','+',' ']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def integer_encode(X,y,alphabet):\n",
    "    char_to_int = dict((c,i) for i,c in enumerate(alphabet))\n",
    "    Xenc = list()\n",
    "    for pattern in X:\n",
    "        integer_encoded = [char_to_int[char] for char in pattern]\n",
    "        Xenc.append(integer_encoded)\n",
    "    yenc = list()\n",
    "    for pattern in y:\n",
    "        integer_encoded = [char_to_int[char] for char in pattern]\n",
    "        yenc.append(integer_encoded)\n",
    "    return Xenc, yenc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = integer_encode(X,y,alphabet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[11, 3, 10, 1, 0]] [[1, 3]]\n"
     ]
    }
   ],
   "source": [
    "print(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(X,y,max_int):\n",
    "    Xenc = list()\n",
    "    for seq in X:\n",
    "        pattern = list()\n",
    "        for index in seq:\n",
    "            vector = [0 for _ in range(max_int)]\n",
    "            vector[index] = 1\n",
    "            pattern.append(vector)\n",
    "        Xenc.append(pattern)\n",
    "        \n",
    "    yenc = list()\n",
    "    for seq in y:\n",
    "        pattern = list()\n",
    "        for index in seq:\n",
    "            vector = [0 for _ in range(max_int)]\n",
    "            vector[index] = 1\n",
    "            pattern.append(vector)\n",
    "        yenc.append(pattern)\n",
    "    return Xenc, yenc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = one_hot_encode(X,y,len(alphabet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]] [[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]]]\n"
     ]
    }
   ],
   "source": [
    "print(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(n_samples,n_numbers, largest, alphabet):\n",
    "    X,y = random_sum_pairs(n_samples,n_numbers,largest)\n",
    "    X,y = to_string(X,y,n_numbers,largest)\n",
    "    X,y = integer_encode(X,y,alphabet)\n",
    "    X,y = one_hot_encode(X,y,len(alphabet))\n",
    "    X,y = array(X), array(y)\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def invert(seq,alphabet):\n",
    "    int_to_char = dict((i,c) for i,c in enumerate(alphabet))\n",
    "    strings  = list()\n",
    "    for pattern in seq:\n",
    "        string = int_to_char[argmax(pattern)]\n",
    "        strings.append(string)\n",
    "    return ''.join(strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_terms = 3\n",
    "largest = 10\n",
    "alphabet = [str(x) for x in range(10)] + ['+', ' ']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_chars = len(alphabet)\n",
    "n_in_seq_length = n_terms*ceil(log10(largest+1)) +n_terms-1\n",
    "n_out_seq_length = ceil(log10(n_terms*(largest+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import RepeatVector\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_3 (LSTM)                (None, 75)                26400     \n",
      "_________________________________________________________________\n",
      "repeat_vector_2 (RepeatVecto (None, 2, 75)             0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 2, 50)             25200     \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (None, 2, 12)             612       \n",
      "=================================================================\n",
      "Total params: 52,212\n",
      "Trainable params: 52,212\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(75, input_shape=(n_in_seq_length,n_chars)))\n",
    "model.add(RepeatVector(n_out_seq_length))\n",
    "model.add(LSTM(50,return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(n_chars,activation='softmax')))\n",
    "model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s 667us/step - loss: 1.9809 - acc: 0.3560\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f4bab1830b8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X,y = generate_data(1000,n_terms,largest,alphabet)\n",
    "model.fit(X,y,epochs=1,batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.642976, Accuracy: 37.500000\n"
     ]
    }
   ],
   "source": [
    "X,y = generate_data(100,n_terms,largest,alphabet)\n",
    "loss,acc = model.evaluate(X,y,verbose=0)\n",
    "print('Loss: %f, Accuracy: %f' %(loss,acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   6+7+6 = 11 (expect 19)\n",
      "  10+6+7 = 11 (expect 23)\n",
      " 9+10+10 = 11 (expect 29)\n",
      "   2+4+7 = 11 (expect 13)\n",
      "   6+9+8 = 11 (expect 23)\n",
      "   6+2+8 = 11 (expect 16)\n",
      "   6+1+4 = 11 (expect 11)\n",
      "   5+7+3 = 11 (expect 15)\n",
      "   4+5+9 = 11 (expect 18)\n",
      "  10+1+3 = 11 (expect 14)\n"
     ]
    }
   ],
   "source": [
    "for _ in range(10):\n",
    "    X,y = generate_data(1,n_terms,largest,alphabet)\n",
    "    yhat = model.predict(X,verbose=0)\n",
    "    in_seq = invert(X[0],alphabet)\n",
    "    out_seq = invert(y[0],alphabet)\n",
    "    predicted = invert(yhat[0],alphabet)\n",
    "    print('%s = %s (expect %s)' %(in_seq,predicted,out_seq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}