copy_training_USE_THIS_ONE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import re\n",
    "import os\n",
    "import sys\n",
    "import tqdm\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "import numpy as np\n",
    "from nltk import tokenize\n",
    "\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.utils import Sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Don't run this cell unless you are using GPU\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "  # Restrict TensorFlow to only use the first GPU\n",
    "  try:\n",
    "    tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPU\")\n",
    "  except RuntimeError as e:\n",
    "    # Visible devices must be set before GPUs have been initialized\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Debug:\n",
    "    def __init__(self, debug_mode=True):\n",
    "        self.debug_mode = debug_mode\n",
    "        self.flag = {}\n",
    "\n",
    "    def log(self, target, flag=None):\n",
    "        if self.debug_mode:\n",
    "            if flag is None:\n",
    "                print(target)\n",
    "            else:\n",
    "                if flag in self.flag.keys():\n",
    "                    if self.flag[flag]:\n",
    "                        print(target)\n",
    "\n",
    "    def set_flag(self, flag: str, val: bool):\n",
    "        self.flag[flag] = val\n",
    "\n",
    "debug = Debug(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class GeneratorExceptions(Exception):\n",
    "    \"\"\"\n",
    "    The Exception class for tracking all exceptions raised in data generator\n",
    "    Param\n",
    "        text: the displayed text\n",
    "    \"\"\"\n",
    "    def __init__(self, text: str):\n",
    "        self.text = text\n",
    "\n",
    "class data_generator(Sequence):\n",
    "    \"\"\"The data generator for the model.\n",
    "    \n",
    "    The data generator that batches and loads data and the pass it into the model\n",
    "    \n",
    "    Attributes:\n",
    "        dataset_file_path:  the path to the dataset file, only need when using our preprocessing method\n",
    "        processed_dataset_path: the path to the processed dataset directory\n",
    "        batch_size: the number of files in each batch, it's recommend to be larger than 1\n",
    "        shuffle: if the generator shuffle the dataset after each epoch\n",
    "        file_list: the list of all processed data\n",
    "        encoder_input_data: the input data for the encoder\n",
    "        decoder_input_data: the input data for the decoder\n",
    "        decoder_input_data: the target data for the decoder\n",
    "        char_list: the list that stores all possible characters in the data\n",
    "        characters_set: the set generated based on the char_lists\n",
    "        tokens_count: total number of all possible characters\n",
    "        max_sequence_len: the maximum length of a sequence\n",
    "    \"\"\"\n",
    "    def __init__(self, \n",
    "                 dataset_file_path : str=\"data/dataset/nysk.xml\", \n",
    "                 processed_dataset_path: str =\"data/processed_dataset/\",\n",
    "                 batch_size = 2,\n",
    "                 shuffle = True):\n",
    "        \"\"\"The constructor.\n",
    "        Args:\n",
    "            dataset_file_path: the path to the dataset file, only need when using our preprocessing method\n",
    "            processed_dataset_path: the path to the processed dataset directory\n",
    "            batch_size: the number of files in each batch, it's recommend to be larger than 1\n",
    "            shuffle: if the generator shuffle the dataset after each epoch\n",
    "        \"\"\"\n",
    "        self.dataset_file_path = dataset_file_path\n",
    "        self.processed_dataset_path = processed_dataset_path\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        \n",
    "        self.file_list = []\n",
    "        \n",
    "        self.encoder_input_data = []\n",
    "        self.decoder_input_data = []\n",
    "        self.decoder_target_data = []\n",
    "        \n",
    "        self.char_list = []\n",
    "        self.characters_set = set()\n",
    "        self.tokens_count = 0\n",
    "        self.max_sequence_len = 0\n",
    "        \n",
    "        self.token_index = None\n",
    "        self.input_texts = None\n",
    "        self.model = None\n",
    "        \n",
    "        self.epoch_count = 0\n",
    "        \n",
    "        self.validation_split=0.1\n",
    "    \n",
    "    def generate_char_dict(self):\n",
    "        \"\"\"Generates the character dictionary (Token index)\"\"\"\n",
    "        for i in string.ascii_letters:\n",
    "            self.characters_set.add(i)\n",
    "        \n",
    "        for i in \"1234567890\\n\\t.,!(){}\\\"\\' \":\n",
    "            self.characters_set.add(i)\n",
    "        \n",
    "        self.char_list = sorted(list(self.characters_set))\n",
    "        self.tokens_count = len(self.char_list)\n",
    "        self.token_index = dict([(char, i) for i, char in enumerate(self.characters_set)])\n",
    "\n",
    "        \n",
    "    def preprocess_data(self, override=False, max:int=-1):\n",
    "        \"\"\"Preprocess the data to enabel to model to use these data\n",
    "        \n",
    "        This function removes the unnecessary characters in the dataset and save each element\n",
    "        into a separated txt file\n",
    "        \n",
    "        Args:\n",
    "            override: if this function overwrite the file when already exists\n",
    "            max: the maximum number of files that will be processed, in case not the whole dataset \n",
    "                 want to be used\n",
    "        \"\"\"\n",
    "        self.file_list = []\n",
    "        if os.path.isfile(self.dataset_file_path):\n",
    "            if not os.path.isdir(self.processed_dataset_path):\n",
    "                os.mkdir(self.processed_dataset_path)\n",
    "\n",
    "            with open(self.dataset_file_path, \"r\", encoding=\"utf-8\") as f:\n",
    "                doc = ET.ElementTree(file=f)\n",
    "\n",
    "            root = doc.getroot()\n",
    "            \n",
    "            pos = 0\n",
    "            \n",
    "            for item in tqdm.tqdm(root):\n",
    "                if max != -1:\n",
    "                    if pos > max:\n",
    "                        break\n",
    "                    pos += 1\n",
    "                news_id = item.findtext('docid')\n",
    "                source = item.findtext('source')\n",
    "                url = item.findtext('url')\n",
    "                title = item.findtext('title')\n",
    "                summary = item.findtext('summary')\n",
    "                raw_text = item.findtext('text')\n",
    "\n",
    "                title = re.sub(r\"<.*>\", \"\", title)\n",
    "                title = re.sub(r\"\\W\", \"_\", title)\n",
    "                title = f\"{news_id}_{title[:10]}\"\n",
    "                \n",
    "                res = tokenize.sent_tokenize(raw_text)\n",
    "                sentences_count = len(res)\n",
    "                       \n",
    "                text = \"\"\n",
    "                \n",
    "                \n",
    "                for s in res:\n",
    "                    \n",
    "                    if len(s) > 200:\n",
    "                        continue\n",
    "                    else:\n",
    "                        if len(s) > self.max_sequence_len:\n",
    "                            self.max_sequence_len = len(s)\n",
    "                        t_sentence = \"\"\n",
    "                        for c in s:\n",
    "                            if c in self.characters_set:\n",
    "                                t_sentence += c\n",
    "                        t_sentence += \"\\t\"\n",
    "                        text += t_sentence\n",
    "                \n",
    "                fp = f\"{self.processed_dataset_path}{title}_{sentences_count}.txt\"\n",
    "                \n",
    "                if not os.path.isfile(fp) or override:\n",
    "                    with open(fp, 'w', encoding='utf-8') as f:\n",
    "                        f.write(text)\n",
    "            self.max_sequence_len += 3\n",
    "            \n",
    "        else:\n",
    "            raise GeneratorExceptions(\"Path doesn't exist\")\n",
    "    \n",
    "    def generate_file_list(self, length:int=-1):\n",
    "        \"\"\"Generated the list of elements\"\"\"\n",
    "        temp = os.listdir(self.processed_dataset_path)\n",
    "        self.file_list = []\n",
    "        t_list = []\n",
    "        for i in temp:\n",
    "            if os.path.splitext(i)[-1] == \".txt\":\n",
    "                t_list.append(f\"{self.processed_dataset_path}{i}\")\n",
    "        self.file_list = t_list[1:length]\n",
    "    \n",
    "    def process_data(self, text):\n",
    "        \"\"\"process and load a file\"\"\"\n",
    "        input_texts = []\n",
    "        target_texts = []\n",
    "        \n",
    "        for i in range(0, len(text)-1):\n",
    "            input_t = f\"\\t{text[i]}\\n\"\n",
    "            target_t = f\"\\t{text[i+1]}\\n\"\n",
    "            input_texts.append(input_t)\n",
    "            target_texts.append(target_t)\n",
    "        \n",
    "        temp_encoder_input_data = np.zeros(\n",
    "            (len(input_texts), self.max_sequence_len, self.tokens_count), dtype=\"float32\"\n",
    "        )\n",
    "        temp_decoder_input_data = np.zeros(\n",
    "            (len(input_texts), self.max_sequence_len, self.tokens_count), dtype=\"float32\"\n",
    "        )\n",
    "        temp_decoder_target_data = np.zeros(\n",
    "            (len(input_texts), self.max_sequence_len, self.tokens_count), dtype=\"float32\"\n",
    "        )\n",
    "\n",
    "        for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
    "            for t, char in enumerate(input_text):\n",
    "                temp_encoder_input_data[i, t, self.token_index[char]] = 1.0\n",
    "                temp_encoder_input_data[i, t + 1 :, self.token_index[\" \"]] = 1.0\n",
    "            for t, char in enumerate(target_text):\n",
    "                # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "                temp_decoder_input_data[i, t, self.token_index[char]] = 1.0\n",
    "                if t > 0:\n",
    "                    # decoder_target_data will be ahead by one timestep\n",
    "                    # and will not include the start character.\n",
    "                    temp_decoder_target_data[i, t - 1, self.token_index[char]] = 1.0\n",
    "                    temp_decoder_input_data[i, t + 1 :, self.token_index[\" \"]] = 1.0\n",
    "                    temp_decoder_target_data[i, t:, self.token_index[\" \"]] = 1.0\n",
    "        \n",
    "        self.encoder_input_data = temp_encoder_input_data\n",
    "        self.decoder_input_data = temp_decoder_input_data\n",
    "        self.decoder_target_data = temp_decoder_target_data\n",
    "        \n",
    "        self.input_texts = input_texts\n",
    "        \n",
    "        return\n",
    "    \n",
    "    # The following methods will be called during the training and shouldn't be manually called\n",
    "    # unless you know the reason you are calling these methods\n",
    "    def __len__(self):\n",
    "        return len(self.file_list) // self.batch_size\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        files = self.file_list[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "        res = []\n",
    "        for fl in files:\n",
    "            with open(fl, 'r', encoding='utf-8') as dt:\n",
    "                text = dt.read()\n",
    "                temp_list = text.split(\"\\t\")\n",
    "                res.extend(temp_list)\n",
    "        \n",
    "        self.process_data(res)\n",
    "        return [self.encoder_input_data, self.decoder_input_data], self.decoder_target_data,\n",
    "    \n",
    "    def on_epoch_end(self):\n",
    "        if self.shuffle:\n",
    "            random.shuffle(self.file_list)\n",
    "        model.save_weights(f\"ckpt/ckpt-{self.epoch_count}.hdf5\")\n",
    "        self.epoch_count += 1\n",
    "        return\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run this when you see exceptions from nltk\n",
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'K': 0, 'F': 1, 'q': 2, 'D': 3, ')': 4, 'a': 5, 'U': 6, 'b': 7, 'o': 8, 'V': 9, '.': 10, 'd': 11, 'N': 12, 'Q': 13, 'h': 14, 'S': 15, 'p': 16, 'i': 17, 'P': 18, 'M': 19, 'A': 20, 'Z': 21, '2': 22, '5': 23, 's': 24, 'u': 25, '0': 26, '(': 27, ',': 28, 'e': 29, 'z': 30, 'G': 31, 'T': 32, '7': 33, '\"': 34, 'E': 35, 'H': 36, '\\t': 37, ' ': 38, 't': 39, 'J': 40, 'Y': 41, 'w': 42, 'I': 43, 'O': 44, 'W': 45, '1': 46, '\\n': 47, '!': 48, 'm': 49, 'B': 50, 'f': 51, 'x': 52, 'X': 53, 'y': 54, 'v': 55, 'n': 56, 'C': 57, '3': 58, 'L': 59, 'R': 60, '{': 61, '6': 62, '4': 63, 'r': 64, 'k': 65, 'c': 66, 'l': 67, '9': 68, 'g': 69, '}': 70, \"'\": 71, '8': 72, 'j': 73}\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and check the data generator\n",
    "DataGenerator = data_generator()\n",
    "\n",
    "# Generate the character dictionary\n",
    "DataGenerator.generate_char_dict()\n",
    "\n",
    "print(DataGenerator.token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 201/10421 [00:00<00:19, 512.44it/s]\n"
     ]
    }
   ],
   "source": [
    "# Preprocess the data\n",
    "DataGenerator.preprocess_data(override=True, max=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74\n",
      "203\n"
     ]
    }
   ],
   "source": [
    "# Check if the data and the data generator is properly generated\n",
    "print(DataGenerator.tokens_count)\n",
    "print(DataGenerator.max_sequence_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n",
      "199\n"
     ]
    }
   ],
   "source": [
    "DataGenerator.generate_file_list()\n",
    "print(len(DataGenerator))\n",
    "print(len(DataGenerator.file_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def generate_model(num_encoder_tokens, num_decoder_tokens, latent_dim=256):\n",
    "    \"\"\"generate the mode based on the given size\"\"\"\n",
    "    encoder_inputs = keras.Input(shape=(None, num_encoder_tokens))\n",
    "    encoder = keras.layers.LSTM(latent_dim, return_state=True)\n",
    "    encoder_outputs_, state_h, state_c = encoder(encoder_inputs)\n",
    "    \n",
    "    encoder_states = [state_h, state_c]\n",
    "    \n",
    "    decoder_inputs = keras.Input(shape=(None, num_decoder_tokens))\n",
    "    \n",
    "    decoder_lstm = keras.layers.LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "    decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states)\n",
    "    decoder_dense = keras.layers.Dense(num_decoder_tokens, activation=\"softmax\")\n",
    "    decoder_outputs = decoder_dense(decoder_outputs)\n",
    "    \n",
    "    model = keras.Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "    \n",
    "    model.summary()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 256), (None, 338944      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, None, 256),  338944      input_2[0][0]                    \n",
      "                                                                 lstm[0][1]                       \n",
      "                                                                 lstm[0][2]                       \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, None, 74)     19018       lstm_1[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 696,906\n",
      "Trainable params: 696,906\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Generates and compiles the model\n",
    "latent_dim = 256\n",
    "\n",
    "model = generate_model(num_encoder_tokens=DataGenerator.tokens_count, \n",
    "                       num_decoder_tokens=DataGenerator.tokens_count,\n",
    "                       latent_dim=latent_dim)\n",
    "model.compile(\n",
    "    optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56\n",
      "2.3.1\n",
      "Num GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "# This cell is used for debugging and testing\n",
    "print(sys.getsizeof(model))\n",
    "# model.load_weights(\"ckpt/ckpt-0005.ckpt\")\n",
    "print(tf.__version__)\n",
    "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'K': 0, 'F': 1, 'q': 2, 'D': 3, ')': 4, 'a': 5, 'U': 6, 'b': 7, 'o': 8, 'V': 9, '.': 10, 'd': 11, 'N': 12, 'Q': 13, 'h': 14, 'S': 15, 'p': 16, 'i': 17, 'P': 18, 'M': 19, 'A': 20, 'Z': 21, '2': 22, '5': 23, 's': 24, 'u': 25, '0': 26, '(': 27, ',': 28, 'e': 29, 'z': 30, 'G': 31, 'T': 32, '7': 33, '\"': 34, 'E': 35, 'H': 36, '\\t': 37, ' ': 38, 't': 39, 'J': 40, 'Y': 41, 'w': 42, 'I': 43, 'O': 44, 'W': 45, '1': 46, '\\n': 47, '!': 48, 'm': 49, 'B': 50, 'f': 51, 'x': 52, 'X': 53, 'y': 54, 'v': 55, 'n': 56, 'C': 57, '3': 58, 'L': 59, 'R': 60, '{': 61, '6': 62, '4': 63, 'r': 64, 'k': 65, 'c': 66, 'l': 67, '9': 68, 'g': 69, '}': 70, \"'\": 71, '8': 72, 'j': 73}\n",
      "Epoch 1/20\n",
      "99/99 [==============================] - 69s 700ms/step - loss: 2.3126 - accuracy: 0.7454\n",
      "Epoch 2/20\n",
      "99/99 [==============================] - 68s 682ms/step - loss: 2.0396 - accuracy: 0.7536\n",
      "Epoch 3/20\n",
      "99/99 [==============================] - 68s 686ms/step - loss: 1.8955 - accuracy: 0.7535\n",
      "Epoch 4/20\n",
      "99/99 [==============================] - 68s 682ms/step - loss: 1.8205 - accuracy: 0.7527\n",
      "Epoch 5/20\n",
      "99/99 [==============================] - 69s 695ms/step - loss: 1.7631 - accuracy: 0.7519\n",
      "Epoch 6/20\n",
      "99/99 [==============================] - 68s 683ms/step - loss: 1.7231 - accuracy: 0.7525\n",
      "Epoch 7/20\n",
      "99/99 [==============================] - 68s 690ms/step - loss: 1.6929 - accuracy: 0.7526\n",
      "Epoch 8/20\n",
      "99/99 [==============================] - 67s 675ms/step - loss: 1.6655 - accuracy: 0.7526\n",
      "Epoch 9/20\n",
      "99/99 [==============================] - 68s 688ms/step - loss: 1.6465 - accuracy: 0.7521\n",
      "Epoch 10/20\n",
      "99/99 [==============================] - 69s 696ms/step - loss: 1.6273 - accuracy: 0.7533\n",
      "Epoch 11/20\n",
      "99/99 [==============================] - 67s 677ms/step - loss: 1.6100 - accuracy: 0.7536\n",
      "Epoch 12/20\n",
      "99/99 [==============================] - 67s 682ms/step - loss: 1.5996 - accuracy: 0.7536\n",
      "Epoch 13/20\n",
      "99/99 [==============================] - 67s 679ms/step - loss: 1.5883 - accuracy: 0.7539\n",
      "Epoch 14/20\n",
      "99/99 [==============================] - 68s 691ms/step - loss: 1.5767 - accuracy: 0.7544\n",
      "Epoch 15/20\n",
      "99/99 [==============================] - 68s 682ms/step - loss: 1.5648 - accuracy: 0.7543\n",
      "Epoch 16/20\n",
      "99/99 [==============================] - 67s 681ms/step - loss: 1.5546 - accuracy: 0.7550\n",
      "Epoch 17/20\n",
      "99/99 [==============================] - 67s 675ms/step - loss: 1.5462 - accuracy: 0.7543\n",
      "Epoch 18/20\n",
      "99/99 [==============================] - 66s 668ms/step - loss: 1.5391 - accuracy: 0.7541\n",
      "Epoch 19/20\n",
      "99/99 [==============================] - 67s 681ms/step - loss: 1.5252 - accuracy: 0.7553\n",
      "Epoch 20/20\n",
      "99/99 [==============================] - 66s 666ms/step - loss: 1.5213 - accuracy: 0.7548\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f624c49e860>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training the model\n",
    "epochs = 20\n",
    "\n",
    "print(DataGenerator.token_index)\n",
    "DataGenerator.model = model\n",
    "\n",
    "model.fit(DataGenerator, \n",
    "          epochs=epochs, \n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Save the weights after the training\n",
    "# All the checkpoints can also be used as weights\n",
    "model.save_weights(\"Model/test_weights.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_17\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_8 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                   [(None, 256), (None, 338944      input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_7 (LSTM)                   [(None, None, 256),  338944      input_8[0][0]                    \n",
      "                                                                 lstm_6[0][1]                     \n",
      "                                                                 lstm_6[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, None, 74)     19018       lstm_7[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 696,906\n",
      "Trainable params: 696,906\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"functional_17\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_8 (InputLayer)            [(None, None, 74)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                   [(None, 256), (None, 338944      input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_7 (LSTM)                   [(None, None, 256),  338944      input_8[0][0]                    \n",
      "                                                                 lstm_6[0][1]                     \n",
      "                                                                 lstm_6[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, None, 74)     19018       lstm_7[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 696,906\n",
      "Trainable params: 696,906\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Please make sure the model size is consistent\n",
    "new_model = generate_model(num_encoder_tokens=DataGenerator.tokens_count, \n",
    "                           num_decoder_tokens=DataGenerator.tokens_count,\n",
    "                           latent_dim=latent_dim)\n",
    "new_model.compile(\n",
    "    optimizer=\"rmsprop\", \n",
    "    loss=\"categorical_crossentropy\", \n",
    "    metrics=[\"accuracy\"]\n",
    ")\n",
    "\n",
    "new_model.summary()\n",
    "\n",
    "# The trained weights are in the Model/ folder, load anything you want\n",
    "# All checkkpoints are in ckpt/ folder\n",
    "\n",
    "new_model.load_weights(\"Model/trained_weights.hdf5\")\n",
    "# new_model.load_weights(\"ckpt/ckpt-0005.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "encoder_inputs = new_model.input[0]  # input_1\n",
    "encoder_outputs, state_h_enc, state_c_enc = new_model.layers[2].output  # lstm_1\n",
    "encoder_states = [state_h_enc, state_c_enc]\n",
    "encoder_model = keras.Model(encoder_inputs, encoder_states)\n",
    "\n",
    "decoder_inputs = new_model.input[1]  # input_2\n",
    "\n",
    "# If there's any issue with conflict of names, change the name of the following two lines\n",
    "decoder_state_input_h = keras.Input(shape=(latent_dim,), name=\"input_5\")\n",
    "decoder_state_input_c = keras.Input(shape=(latent_dim,), name=\"input_6\")\n",
    "\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "decoder_lstm = new_model.layers[3]\n",
    "decoder_outputs, state_h_dec, state_c_dec = decoder_lstm(\n",
    "    decoder_inputs, initial_state=decoder_states_inputs\n",
    ")\n",
    "decoder_states = [state_h_dec, state_c_dec]\n",
    "decoder_dense = new_model.layers[4]\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "decoder_model = keras.Model(\n",
    "    [decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states\n",
    ")\n",
    "\n",
    "reverse_input_char_index = dict((i, char) for char, i in DataGenerator.token_index.items())\n",
    "reverse_target_char_index = dict((i, char) for char, i in DataGenerator.token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(DataGenerator: data_generator, input_seq):\n",
    "    \"\"\"function for decoding the generated sequence\"\"\"\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "\n",
    "    target_seq = np.zeros((1, 1, DataGenerator.tokens_count))\n",
    "    target_seq[0, 0, DataGenerator.token_index[\"\\t\"]] = 1.0\n",
    "\n",
    "    stop_condition = False\n",
    "    decoded_sentence = \"\"\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
    "\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += sampled_char\n",
    "\n",
    "        if sampled_char == \"\\n\" or len(decoded_sentence) > DataGenerator.max_sequence_len:\n",
    "            stop_condition = True\n",
    "\n",
    "        target_seq = np.zeros((1, 1, DataGenerator.tokens_count))\n",
    "        target_seq[0, 0, sampled_token_index] = 1.0\n",
    "\n",
    "        states_value = [h, c]\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "-\n",
      "Input sentence: \tIf you work in a corporate environment you may not be able to upgrade your browser to IE 7 or IE8.\n",
      "\n",
      "Decoded sentence: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n"
     ]
    }
   ],
   "source": [
    "# Predict and output the result\n",
    "for seq_index in range(1):\n",
    "    print(seq_index)\n",
    "    DataGenerator.__getitem__(0)\n",
    "    input_seq = DataGenerator.encoder_input_data[seq_index : seq_index + 1]\n",
    "    decoded_sentence = decode_sequence(DataGenerator, input_seq)\n",
    "    print(\"-\")\n",
    "    print(\"Input sentence:\", DataGenerator.input_texts[seq_index])\n",
    "\n",
    "    print(\"Decoded sentence:\", decoded_sentence)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}