TweetsSentimentPredictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.032369,
     "end_time": "2020-10-06T21:19:31.851445",
     "exception": false,
     "start_time": "2020-10-06T21:19:31.819076",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Overview\n",
    "\n",
    "This project is based on the dataset available at https://www.kaggle.com/c/tweet-sentiment-extraction/overview which is composed of about 20k tweets to train sentiment predictors.\n",
    "\n",
    "This notebook will guide you through the process of tweets cleaning (a very basic NLP task when dealing with text data), training a few Deep Learning models with different architectures and finally inferencing on test text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030876,
     "end_time": "2020-10-06T21:19:31.915205",
     "exception": false,
     "start_time": "2020-10-06T21:19:31.884329",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:31.986613Z",
     "iopub.status.busy": "2020-10-06T21:19:31.985943Z",
     "iopub.status.idle": "2020-10-06T21:19:41.514922Z",
     "shell.execute_reply": "2020-10-06T21:19:41.515686Z"
    },
    "papermill": {
     "duration": 9.569887,
     "end_time": "2020-10-06T21:19:41.515898",
     "exception": false,
     "start_time": "2020-10-06T21:19:31.946011",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
    "from collections import Counter\n",
    "from wordcloud import WordCloud\n",
    "from gensim.utils import simple_preprocess\n",
    "import gensim\n",
    "from sklearn.model_selection import train_test_split\n",
    "import spacy\n",
    "import pickle\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.032245,
     "end_time": "2020-10-06T21:19:41.583484",
     "exception": false,
     "start_time": "2020-10-06T21:19:41.551239",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data importing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:41.667156Z",
     "iopub.status.busy": "2020-10-06T21:19:41.666425Z",
     "iopub.status.idle": "2020-10-06T21:19:41.753917Z",
     "shell.execute_reply": "2020-10-06T21:19:41.752384Z"
    },
    "papermill": {
     "duration": 0.132806,
     "end_time": "2020-10-06T21:19:41.754037",
     "exception": false,
     "start_time": "2020-10-06T21:19:41.621231",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.031706,
     "end_time": "2020-10-06T21:19:41.817762",
     "exception": false,
     "start_time": "2020-10-06T21:19:41.786056",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:41.898583Z",
     "iopub.status.busy": "2020-10-06T21:19:41.897942Z",
     "iopub.status.idle": "2020-10-06T21:19:41.908219Z",
     "shell.execute_reply": "2020-10-06T21:19:41.908804Z"
    },
    "papermill": {
     "duration": 0.05902,
     "end_time": "2020-10-06T21:19:41.908931",
     "exception": false,
     "start_time": "2020-10-06T21:19:41.849911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>text</th>\n",
       "      <th>selected_text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cb774db0d1</td>\n",
       "      <td>I`d have responded, if I were going</td>\n",
       "      <td>I`d have responded, if I were going</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>549e992a42</td>\n",
       "      <td>Sooo SAD I will miss you here in San Diego!!!</td>\n",
       "      <td>Sooo SAD</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>088c60f138</td>\n",
       "      <td>my boss is bullying me...</td>\n",
       "      <td>bullying me</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9642c003ef</td>\n",
       "      <td>what interview! leave me alone</td>\n",
       "      <td>leave me alone</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>358bd9e861</td>\n",
       "      <td>Sons of ****, why couldn`t they put them on t...</td>\n",
       "      <td>Sons of ****,</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28b57f3990</td>\n",
       "      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n",
       "      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6e0c6d75b1</td>\n",
       "      <td>2am feedings for the baby are fun when he is a...</td>\n",
       "      <td>fun</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>50e14c0bb8</td>\n",
       "      <td>Soooo high</td>\n",
       "      <td>Soooo high</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>e050245fbd</td>\n",
       "      <td>Both of you</td>\n",
       "      <td>Both of you</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>fc2cbefa9d</td>\n",
       "      <td>Journey!? Wow... u just became cooler.  hehe....</td>\n",
       "      <td>Wow... u just became cooler.</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2339a9b08b</td>\n",
       "      <td>as much as i love to be hopeful, i reckon the...</td>\n",
       "      <td>as much as i love to be hopeful, i reckon the ...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>16fab9f95b</td>\n",
       "      <td>I really really like the song Love Story by Ta...</td>\n",
       "      <td>like</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>74a76f6e0a</td>\n",
       "      <td>My Sharpie is running DANGERously low on ink</td>\n",
       "      <td>DANGERously</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>04dd1d2e34</td>\n",
       "      <td>i want to go to music tonight but i lost my vo...</td>\n",
       "      <td>lost</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>bbe3cbf620</td>\n",
       "      <td>test test from the LG enV2</td>\n",
       "      <td>test test from the LG enV2</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        textID                                               text  \\\n",
       "0   cb774db0d1                I`d have responded, if I were going   \n",
       "1   549e992a42      Sooo SAD I will miss you here in San Diego!!!   \n",
       "2   088c60f138                          my boss is bullying me...   \n",
       "3   9642c003ef                     what interview! leave me alone   \n",
       "4   358bd9e861   Sons of ****, why couldn`t they put them on t...   \n",
       "5   28b57f3990  http://www.dothebouncy.com/smf - some shameles...   \n",
       "6   6e0c6d75b1  2am feedings for the baby are fun when he is a...   \n",
       "7   50e14c0bb8                                         Soooo high   \n",
       "8   e050245fbd                                        Both of you   \n",
       "9   fc2cbefa9d   Journey!? Wow... u just became cooler.  hehe....   \n",
       "10  2339a9b08b   as much as i love to be hopeful, i reckon the...   \n",
       "11  16fab9f95b  I really really like the song Love Story by Ta...   \n",
       "12  74a76f6e0a       My Sharpie is running DANGERously low on ink   \n",
       "13  04dd1d2e34  i want to go to music tonight but i lost my vo...   \n",
       "14  bbe3cbf620                         test test from the LG enV2   \n",
       "\n",
       "                                        selected_text sentiment  \n",
       "0                 I`d have responded, if I were going   neutral  \n",
       "1                                            Sooo SAD  negative  \n",
       "2                                         bullying me  negative  \n",
       "3                                      leave me alone  negative  \n",
       "4                                       Sons of ****,  negative  \n",
       "5   http://www.dothebouncy.com/smf - some shameles...   neutral  \n",
       "6                                                 fun  positive  \n",
       "7                                          Soooo high   neutral  \n",
       "8                                         Both of you   neutral  \n",
       "9                        Wow... u just became cooler.  positive  \n",
       "10  as much as i love to be hopeful, i reckon the ...   neutral  \n",
       "11                                               like  positive  \n",
       "12                                        DANGERously  negative  \n",
       "13                                               lost  negative  \n",
       "14                         test test from the LG enV2   neutral  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:41.980569Z",
     "iopub.status.busy": "2020-10-06T21:19:41.979730Z",
     "iopub.status.idle": "2020-10-06T21:19:41.983566Z",
     "shell.execute_reply": "2020-10-06T21:19:41.983073Z"
    },
    "papermill": {
     "duration": 0.041264,
     "end_time": "2020-10-06T21:19:41.983666",
     "exception": false,
     "start_time": "2020-10-06T21:19:41.942402",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27481"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's get the dataset lenght\n",
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.060070Z",
     "iopub.status.busy": "2020-10-06T21:19:42.059141Z",
     "iopub.status.idle": "2020-10-06T21:19:42.064587Z",
     "shell.execute_reply": "2020-10-06T21:19:42.064115Z"
    },
    "papermill": {
     "duration": 0.047577,
     "end_time": "2020-10-06T21:19:42.064691",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.017114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['neutral', 'negative', 'positive'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Is there any other different value than neutral, negative and positive?\n",
    "train['sentiment'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.143106Z",
     "iopub.status.busy": "2020-10-06T21:19:42.142318Z",
     "iopub.status.idle": "2020-10-06T21:19:42.185676Z",
     "shell.execute_reply": "2020-10-06T21:19:42.186310Z"
    },
    "papermill": {
     "duration": 0.088183,
     "end_time": "2020-10-06T21:19:42.186453",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.098270",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>text</th>\n",
       "      <th>selected_text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentiment</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>negative</th>\n",
       "      <td>7781</td>\n",
       "      <td>7781</td>\n",
       "      <td>5861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>11118</td>\n",
       "      <td>11117</td>\n",
       "      <td>11111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>positive</th>\n",
       "      <td>8582</td>\n",
       "      <td>8582</td>\n",
       "      <td>5537</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           textID   text  selected_text\n",
       "sentiment                              \n",
       "negative     7781   7781           5861\n",
       "neutral     11118  11117          11111\n",
       "positive     8582   8582           5537"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#How's distributed the dataset? Is it biased?\n",
    "train.groupby('sentiment').nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.033994,
     "end_time": "2020-10-06T21:19:42.255666",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.221672",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data cleaning\n",
    "\n",
    "Even when the dataset is a little bit biased, we'll keep it this way because the differences are not significant."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.334657Z",
     "iopub.status.busy": "2020-10-06T21:19:42.333827Z",
     "iopub.status.idle": "2020-10-06T21:19:42.337209Z",
     "shell.execute_reply": "2020-10-06T21:19:42.337696Z"
    },
    "papermill": {
     "duration": 0.04812,
     "end_time": "2020-10-06T21:19:42.337823",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.289703",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>selected_text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I`d have responded, if I were going</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sooo SAD</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bullying me</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>leave me alone</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sons of ****,</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         selected_text sentiment\n",
       "0  I`d have responded, if I were going   neutral\n",
       "1                             Sooo SAD  negative\n",
       "2                          bullying me  negative\n",
       "3                       leave me alone  negative\n",
       "4                        Sons of ****,  negative"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's keep only the columns that we're going to use\n",
    "train = train[['selected_text','sentiment']]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.415166Z",
     "iopub.status.busy": "2020-10-06T21:19:42.414365Z",
     "iopub.status.idle": "2020-10-06T21:19:42.418112Z",
     "shell.execute_reply": "2020-10-06T21:19:42.417659Z"
    },
    "papermill": {
     "duration": 0.0451,
     "end_time": "2020-10-06T21:19:42.418205",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.373105",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Is there any null value?\n",
    "train[\"selected_text\"].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.499946Z",
     "iopub.status.busy": "2020-10-06T21:19:42.498153Z",
     "iopub.status.idle": "2020-10-06T21:19:42.500760Z",
     "shell.execute_reply": "2020-10-06T21:19:42.501276Z"
    },
    "papermill": {
     "duration": 0.047154,
     "end_time": "2020-10-06T21:19:42.501409",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.454255",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Let's fill the only null value.\n",
    "train[\"selected_text\"].fillna(\"No content\", inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.035753,
     "end_time": "2020-10-06T21:19:42.572975",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.537222",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### The next steps about data cleaning will be:\n",
    "\n",
    "* Remove URLs from the tweets\n",
    "* Tokenize text\n",
    "* Remove emails\n",
    "* Remove new lines characters\n",
    "* Remove distracting single quotes\n",
    "* Remove all punctuation signs\n",
    "* Lowercase all text\n",
    "* Detokenize text\n",
    "* Convert list of texts to Numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.651469Z",
     "iopub.status.busy": "2020-10-06T21:19:42.650518Z",
     "iopub.status.idle": "2020-10-06T21:19:42.653091Z",
     "shell.execute_reply": "2020-10-06T21:19:42.653513Z"
    },
    "papermill": {
     "duration": 0.044735,
     "end_time": "2020-10-06T21:19:42.653654",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.608919",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def depure_data(data):\n",
    "    \n",
    "    #Removing URLs with a regular expression\n",
    "    url_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "    data = url_pattern.sub(r'', data)\n",
    "\n",
    "    # Remove Emails\n",
    "    data = re.sub('\\S*@\\S*\\s?', '', data)\n",
    "\n",
    "    # Remove new line characters\n",
    "    data = re.sub('\\s+', ' ', data)\n",
    "\n",
    "    # Remove distracting single quotes\n",
    "    data = re.sub(\"\\'\", \"\", data)\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:42.742866Z",
     "iopub.status.busy": "2020-10-06T21:19:42.737799Z",
     "iopub.status.idle": "2020-10-06T21:19:43.099846Z",
     "shell.execute_reply": "2020-10-06T21:19:43.100317Z"
    },
    "papermill": {
     "duration": 0.411166,
     "end_time": "2020-10-06T21:19:43.100446",
     "exception": false,
     "start_time": "2020-10-06T21:19:42.689280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I`d have responded, if I were going',\n",
       " 'Sooo SAD',\n",
       " 'bullying me',\n",
       " 'leave me alone',\n",
       " 'Sons of ****,']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = []\n",
    "#Splitting pd.Series to list\n",
    "data_to_list = train['selected_text'].values.tolist()\n",
    "for i in range(len(data_to_list)):\n",
    "    temp.append(depure_data(data_to_list[i]))\n",
    "list(temp[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:43.217126Z",
     "iopub.status.busy": "2020-10-06T21:19:43.201831Z",
     "iopub.status.idle": "2020-10-06T21:19:44.301157Z",
     "shell.execute_reply": "2020-10-06T21:19:44.300459Z"
    },
    "papermill": {
     "duration": 1.161143,
     "end_time": "2020-10-06T21:19:44.301321",
     "exception": false,
     "start_time": "2020-10-06T21:19:43.140178",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['have', 'responded', 'if', 'were', 'going'], ['sooo', 'sad'], ['bullying', 'me'], ['leave', 'me', 'alone'], ['sons', 'of'], ['some', 'shameless', 'plugging', 'for', 'the', 'best', 'rangers', 'forum', 'on', 'earth'], ['fun'], ['soooo', 'high'], ['both', 'of', 'you'], ['wow', 'just', 'became', 'cooler']]\n"
     ]
    }
   ],
   "source": [
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations\n",
    "        \n",
    "\n",
    "data_words = list(sent_to_words(temp))\n",
    "\n",
    "print(data_words[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:44.381205Z",
     "iopub.status.busy": "2020-10-06T21:19:44.380312Z",
     "iopub.status.idle": "2020-10-06T21:19:44.384281Z",
     "shell.execute_reply": "2020-10-06T21:19:44.383753Z"
    },
    "papermill": {
     "duration": 0.045481,
     "end_time": "2020-10-06T21:19:44.384376",
     "exception": false,
     "start_time": "2020-10-06T21:19:44.338895",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27481"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:44.465126Z",
     "iopub.status.busy": "2020-10-06T21:19:44.464351Z",
     "iopub.status.idle": "2020-10-06T21:19:44.467376Z",
     "shell.execute_reply": "2020-10-06T21:19:44.466918Z"
    },
    "papermill": {
     "duration": 0.045096,
     "end_time": "2020-10-06T21:19:44.467470",
     "exception": false,
     "start_time": "2020-10-06T21:19:44.422374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def detokenize(text):\n",
    "    return TreebankWordDetokenizer().detokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:44.561022Z",
     "iopub.status.busy": "2020-10-06T21:19:44.550825Z",
     "iopub.status.idle": "2020-10-06T21:19:47.831584Z",
     "shell.execute_reply": "2020-10-06T21:19:47.832620Z"
    },
    "papermill": {
     "duration": 3.326682,
     "end_time": "2020-10-06T21:19:47.832800",
     "exception": false,
     "start_time": "2020-10-06T21:19:44.506118",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['have responded if were going', 'sooo sad', 'bullying me', 'leave me alone', 'sons of']\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for i in range(len(data_words)):\n",
    "    data.append(detokenize(data_words[i]))\n",
    "print(data[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:47.936780Z",
     "iopub.status.busy": "2020-10-06T21:19:47.934885Z",
     "iopub.status.idle": "2020-10-06T21:19:47.937552Z",
     "shell.execute_reply": "2020-10-06T21:19:47.938090Z"
    },
    "papermill": {
     "duration": 0.06542,
     "end_time": "2020-10-06T21:19:47.938217",
     "exception": false,
     "start_time": "2020-10-06T21:19:47.872797",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = np.array(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.037183,
     "end_time": "2020-10-06T21:19:48.013459",
     "exception": false,
     "start_time": "2020-10-06T21:19:47.976276",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Label encoding\n",
    "\n",
    "As the dataset is categorical, we need to convert the sentiment labels from Neutral, Negative and Positive to a float type that our model can understand. To achieve this task, we'll implement the to_categorical method from Keras."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:48.122784Z",
     "iopub.status.busy": "2020-10-06T21:19:48.121518Z",
     "iopub.status.idle": "2020-10-06T21:19:48.125042Z",
     "shell.execute_reply": "2020-10-06T21:19:48.125481Z"
    },
    "papermill": {
     "duration": 0.074223,
     "end_time": "2020-10-06T21:19:48.125627",
     "exception": false,
     "start_time": "2020-10-06T21:19:48.051404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "labels = np.array(train['sentiment'])\n",
    "y = []\n",
    "for i in range(len(labels)):\n",
    "    if labels[i] == 'neutral':\n",
    "        y.append(0)\n",
    "    if labels[i] == 'negative':\n",
    "        y.append(1)\n",
    "    if labels[i] == 'positive':\n",
    "        y.append(2)\n",
    "y = np.array(y)\n",
    "labels = tf.keras.utils.to_categorical(y, 3, dtype=\"float32\")\n",
    "del y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:48.206863Z",
     "iopub.status.busy": "2020-10-06T21:19:48.206068Z",
     "iopub.status.idle": "2020-10-06T21:19:48.209920Z",
     "shell.execute_reply": "2020-10-06T21:19:48.209425Z"
    },
    "papermill": {
     "duration": 0.046647,
     "end_time": "2020-10-06T21:19:48.210021",
     "exception": false,
     "start_time": "2020-10-06T21:19:48.163374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27481"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.038298,
     "end_time": "2020-10-06T21:19:48.287312",
     "exception": false,
     "start_time": "2020-10-06T21:19:48.249014",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data sequencing and splitting\n",
    "\n",
    "We'll implement the Keras tokenizer as well as its pad_sequences method to transform our text data into 3D float data, otherwise our neural networks won't be able to be trained on it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:48.373707Z",
     "iopub.status.busy": "2020-10-06T21:19:48.372469Z",
     "iopub.status.idle": "2020-10-06T21:19:49.386819Z",
     "shell.execute_reply": "2020-10-06T21:19:49.387562Z"
    },
    "papermill": {
     "duration": 1.062302,
     "end_time": "2020-10-06T21:19:49.387739",
     "exception": false,
     "start_time": "2020-10-06T21:19:48.325437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0    0    0 ...   68  146   41]\n",
      " [   0    0    0 ...    0  397   65]\n",
      " [   0    0    0 ...    0    0   11]\n",
      " ...\n",
      " [   0    0    0 ...  372   10    3]\n",
      " [   0    0    0 ...   24  542    4]\n",
      " [   0    0    0 ... 2424  199  657]]\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import regularizers\n",
    "from keras import backend as K\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "max_words = 5000\n",
    "max_len = 200\n",
    "\n",
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "tokenizer.fit_on_texts(data)\n",
    "sequences = tokenizer.texts_to_sequences(data)\n",
    "tweets = pad_sequences(sequences, maxlen=max_len)\n",
    "print(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:49.478734Z",
     "iopub.status.busy": "2020-10-06T21:19:49.477823Z",
     "iopub.status.idle": "2020-10-06T21:19:49.482217Z",
     "shell.execute_reply": "2020-10-06T21:19:49.483329Z"
    },
    "papermill": {
     "duration": 0.054099,
     "end_time": "2020-10-06T21:19:49.483550",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.429451",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0.]\n",
      " [0. 1. 0.]\n",
      " [0. 1. 0.]\n",
      " ...\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [1. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:49.585814Z",
     "iopub.status.busy": "2020-10-06T21:19:49.584576Z",
     "iopub.status.idle": "2020-10-06T21:19:49.600360Z",
     "shell.execute_reply": "2020-10-06T21:19:49.601002Z"
    },
    "papermill": {
     "duration": 0.069178,
     "end_time": "2020-10-06T21:19:49.601136",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.531958",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20610 6871 20610 6871\n"
     ]
    }
   ],
   "source": [
    "#Splitting the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(tweets,labels, random_state=0)\n",
    "print (len(X_train),len(X_test),len(y_train),len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.044723,
     "end_time": "2020-10-06T21:19:49.690338",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.645615",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model building\n",
    "\n",
    "Alright, in the next cells I'll guide you through the process of building 2 Recurrent Neural Networks. I'll implement sequential models from the Keras API to achieve this task. Essentially, I'll start with a single layer **LSTM** network which is known by achieving good results in NLP tasks when the dataset is relatively small (I could have started with a SimpleRNN which is even simpler, but to be honest it's actually not deployed in production environments because it is too simple - however I'll leave it commented in case you want to know it's built). The next one will be a Bidirectional LSTM model, a more complex one and this particular one is known to achieve great metrics when talking about text classification. If everything goes ok, we should get the best results with the BidRNN, let's see what happens.\n",
    "\n",
    "Let's get hands on:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.045363,
     "end_time": "2020-10-06T21:19:49.779234",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.733871",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## SimpleRNN model (Bonus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:49.876465Z",
     "iopub.status.busy": "2020-10-06T21:19:49.875466Z",
     "iopub.status.idle": "2020-10-06T21:19:49.878789Z",
     "shell.execute_reply": "2020-10-06T21:19:49.878237Z"
    },
    "papermill": {
     "duration": 0.054472,
     "end_time": "2020-10-06T21:19:49.878902",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.824430",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#model0 = Sequential()\n",
    "#model0.add(layers.Embedding(max_words, 15))\n",
    "#model0.add(layers.SimpleRNN(15))\n",
    "#model0.add(layers.Dense(3,activation='softmax'))\n",
    "\n",
    "\n",
    "#model0.compile(optimizer='rmsprop',loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "#Implementing model checkpoins to save the best metric and do not lose it on training.\n",
    "#checkpoint0 = ModelCheckpoint(\"best_model0.hdf5\", monitor='val_accuracy', verbose=1,save_best_only=True, mode='auto', period=1,save_weights_only=False)\n",
    "#history = model0.fit(X_train, y_train, epochs=5,validation_data=(X_test, y_test),callbacks=[checkpoint0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.044937,
     "end_time": "2020-10-06T21:19:49.968644",
     "exception": false,
     "start_time": "2020-10-06T21:19:49.923707",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Single LSTM layer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:19:50.068632Z",
     "iopub.status.busy": "2020-10-06T21:19:50.067909Z",
     "iopub.status.idle": "2020-10-06T21:30:02.803265Z",
     "shell.execute_reply": "2020-10-06T21:30:02.805203Z"
    },
    "papermill": {
     "duration": 612.793273,
     "end_time": "2020-10-06T21:30:02.806611",
     "exception": false,
     "start_time": "2020-10-06T21:19:50.013338",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.8372 - accuracy: 0.6071\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.70179, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.8371 - accuracy: 0.6071 - val_loss: 0.7264 - val_accuracy: 0.7018\n",
      "Epoch 2/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.6427 - accuracy: 0.7456\n",
      "Epoch 00002: val_accuracy improved from 0.70179 to 0.78227, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.6421 - accuracy: 0.7459 - val_loss: 0.5718 - val_accuracy: 0.7823\n",
      "Epoch 3/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.5420 - accuracy: 0.7849\n",
      "Epoch 00003: val_accuracy did not improve from 0.78227\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.5417 - accuracy: 0.7851 - val_loss: 0.5509 - val_accuracy: 0.7709\n",
      "Epoch 4/70\n",
      "640/645 [============================>.] - ETA: 0s - loss: 0.4959 - accuracy: 0.8069\n",
      "Epoch 00004: val_accuracy improved from 0.78227 to 0.81196, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.4962 - accuracy: 0.8068 - val_loss: 0.4889 - val_accuracy: 0.8120\n",
      "Epoch 5/70\n",
      "640/645 [============================>.] - ETA: 0s - loss: 0.4693 - accuracy: 0.8201\n",
      "Epoch 00005: val_accuracy improved from 0.81196 to 0.81415, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.4687 - accuracy: 0.8204 - val_loss: 0.4782 - val_accuracy: 0.8141\n",
      "Epoch 6/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.4510 - accuracy: 0.8276\n",
      "Epoch 00006: val_accuracy improved from 0.81415 to 0.81458, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.4511 - accuracy: 0.8276 - val_loss: 0.4733 - val_accuracy: 0.8146\n",
      "Epoch 7/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.4421 - accuracy: 0.8299\n",
      "Epoch 00007: val_accuracy improved from 0.81458 to 0.82317, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.4424 - accuracy: 0.8298 - val_loss: 0.4662 - val_accuracy: 0.8232\n",
      "Epoch 8/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.4285 - accuracy: 0.8395\n",
      "Epoch 00008: val_accuracy improved from 0.82317 to 0.82419, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.4285 - accuracy: 0.8395 - val_loss: 0.4581 - val_accuracy: 0.8242\n",
      "Epoch 9/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.4195 - accuracy: 0.8396\n",
      "Epoch 00009: val_accuracy improved from 0.82419 to 0.82768, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.4194 - accuracy: 0.8397 - val_loss: 0.4552 - val_accuracy: 0.8277\n",
      "Epoch 10/70\n",
      "640/645 [============================>.] - ETA: 0s - loss: 0.4104 - accuracy: 0.8446\n",
      "Epoch 00010: val_accuracy did not improve from 0.82768\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.4105 - accuracy: 0.8445 - val_loss: 0.4487 - val_accuracy: 0.8242\n",
      "Epoch 11/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4037 - accuracy: 0.8489\n",
      "Epoch 00011: val_accuracy did not improve from 0.82768\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.4037 - accuracy: 0.8489 - val_loss: 0.4595 - val_accuracy: 0.8275\n",
      "Epoch 12/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3959 - accuracy: 0.8518\n",
      "Epoch 00012: val_accuracy improved from 0.82768 to 0.82841, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3954 - accuracy: 0.8521 - val_loss: 0.4495 - val_accuracy: 0.8284\n",
      "Epoch 13/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3892 - accuracy: 0.8563\n",
      "Epoch 00013: val_accuracy improved from 0.82841 to 0.83103, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.3893 - accuracy: 0.8562 - val_loss: 0.4419 - val_accuracy: 0.8310\n",
      "Epoch 14/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3841 - accuracy: 0.8560\n",
      "Epoch 00014: val_accuracy improved from 0.83103 to 0.83423, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3839 - accuracy: 0.8561 - val_loss: 0.4453 - val_accuracy: 0.8342\n",
      "Epoch 15/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3825 - accuracy: 0.8571\n",
      "Epoch 00015: val_accuracy did not improve from 0.83423\n",
      "645/645 [==============================] - 7s 12ms/step - loss: 0.3825 - accuracy: 0.8571 - val_loss: 0.4371 - val_accuracy: 0.8336\n",
      "Epoch 16/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3740 - accuracy: 0.8610\n",
      "Epoch 00016: val_accuracy improved from 0.83423 to 0.83496, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.3739 - accuracy: 0.8609 - val_loss: 0.4333 - val_accuracy: 0.8350\n",
      "Epoch 17/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3709 - accuracy: 0.8617\n",
      "Epoch 00017: val_accuracy improved from 0.83496 to 0.83714, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 12ms/step - loss: 0.3709 - accuracy: 0.8617 - val_loss: 0.4359 - val_accuracy: 0.8371\n",
      "Epoch 18/70\n",
      "640/645 [============================>.] - ETA: 0s - loss: 0.3674 - accuracy: 0.8624\n",
      "Epoch 00018: val_accuracy did not improve from 0.83714\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3677 - accuracy: 0.8624 - val_loss: 0.4353 - val_accuracy: 0.8344\n",
      "Epoch 19/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3626 - accuracy: 0.8669\n",
      "Epoch 00019: val_accuracy did not improve from 0.83714\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3626 - accuracy: 0.8669 - val_loss: 0.4304 - val_accuracy: 0.8358\n",
      "Epoch 20/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3622 - accuracy: 0.8646\n",
      "Epoch 00020: val_accuracy did not improve from 0.83714\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3622 - accuracy: 0.8646 - val_loss: 0.4337 - val_accuracy: 0.8360\n",
      "Epoch 21/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3578 - accuracy: 0.8676\n",
      "Epoch 00021: val_accuracy improved from 0.83714 to 0.83962, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 11ms/step - loss: 0.3580 - accuracy: 0.8674 - val_loss: 0.4343 - val_accuracy: 0.8396\n",
      "Epoch 22/70\n",
      "640/645 [============================>.] - ETA: 0s - loss: 0.3598 - accuracy: 0.8661\n",
      "Epoch 00022: val_accuracy did not improve from 0.83962\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3595 - accuracy: 0.8662 - val_loss: 0.4329 - val_accuracy: 0.8386\n",
      "Epoch 23/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3501 - accuracy: 0.8701\n",
      "Epoch 00023: val_accuracy improved from 0.83962 to 0.84078, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 7s 12ms/step - loss: 0.3501 - accuracy: 0.8701 - val_loss: 0.4257 - val_accuracy: 0.8408\n",
      "Epoch 24/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3496 - accuracy: 0.8696\n",
      "Epoch 00024: val_accuracy did not improve from 0.84078\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3496 - accuracy: 0.8696 - val_loss: 0.4398 - val_accuracy: 0.8379\n",
      "Epoch 25/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3487 - accuracy: 0.8709\n",
      "Epoch 00025: val_accuracy improved from 0.84078 to 0.84296, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3487 - accuracy: 0.8709 - val_loss: 0.4258 - val_accuracy: 0.8430\n",
      "Epoch 26/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3475 - accuracy: 0.8710\n",
      "Epoch 00026: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3476 - accuracy: 0.8709 - val_loss: 0.4332 - val_accuracy: 0.8393\n",
      "Epoch 27/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3471 - accuracy: 0.8727\n",
      "Epoch 00027: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 9s 13ms/step - loss: 0.3470 - accuracy: 0.8727 - val_loss: 0.4314 - val_accuracy: 0.8411\n",
      "Epoch 28/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3471 - accuracy: 0.8724\n",
      "Epoch 00028: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3473 - accuracy: 0.8723 - val_loss: 0.4212 - val_accuracy: 0.8424\n",
      "Epoch 29/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3438 - accuracy: 0.8736\n",
      "Epoch 00029: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3442 - accuracy: 0.8735 - val_loss: 0.4285 - val_accuracy: 0.8414\n",
      "Epoch 30/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3419 - accuracy: 0.8744\n",
      "Epoch 00030: val_accuracy improved from 0.84296 to 0.84398, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3419 - accuracy: 0.8744 - val_loss: 0.4247 - val_accuracy: 0.8440\n",
      "Epoch 31/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3381 - accuracy: 0.8745\n",
      "Epoch 00031: val_accuracy did not improve from 0.84398\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3382 - accuracy: 0.8745 - val_loss: 0.4253 - val_accuracy: 0.8421\n",
      "Epoch 32/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3424 - accuracy: 0.8728\n",
      "Epoch 00032: val_accuracy did not improve from 0.84398\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3426 - accuracy: 0.8728 - val_loss: 0.4262 - val_accuracy: 0.8431\n",
      "Epoch 33/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3411 - accuracy: 0.8758\n",
      "Epoch 00033: val_accuracy did not improve from 0.84398\n",
      "645/645 [==============================] - 9s 13ms/step - loss: 0.3411 - accuracy: 0.8758 - val_loss: 0.4217 - val_accuracy: 0.8424\n",
      "Epoch 34/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3352 - accuracy: 0.8758\n",
      "Epoch 00034: val_accuracy did not improve from 0.84398\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3353 - accuracy: 0.8757 - val_loss: 0.4231 - val_accuracy: 0.8430\n",
      "Epoch 35/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3320 - accuracy: 0.8786\n",
      "Epoch 00035: val_accuracy did not improve from 0.84398\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3317 - accuracy: 0.8787 - val_loss: 0.4316 - val_accuracy: 0.8399\n",
      "Epoch 36/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3352 - accuracy: 0.8773\n",
      "Epoch 00036: val_accuracy improved from 0.84398 to 0.84427, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3352 - accuracy: 0.8773 - val_loss: 0.4276 - val_accuracy: 0.8443\n",
      "Epoch 37/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3328 - accuracy: 0.8763\n",
      "Epoch 00037: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3328 - accuracy: 0.8763 - val_loss: 0.4265 - val_accuracy: 0.8418\n",
      "Epoch 38/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3347 - accuracy: 0.8756\n",
      "Epoch 00038: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3342 - accuracy: 0.8757 - val_loss: 0.4313 - val_accuracy: 0.8419\n",
      "Epoch 39/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3295 - accuracy: 0.8779\n",
      "Epoch 00039: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3297 - accuracy: 0.8777 - val_loss: 0.4294 - val_accuracy: 0.8415\n",
      "Epoch 40/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3316 - accuracy: 0.8777\n",
      "Epoch 00040: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3313 - accuracy: 0.8777 - val_loss: 0.4291 - val_accuracy: 0.8438\n",
      "Epoch 41/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3327 - accuracy: 0.8775\n",
      "Epoch 00041: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3327 - accuracy: 0.8774 - val_loss: 0.4272 - val_accuracy: 0.8402\n",
      "Epoch 42/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3295 - accuracy: 0.8779\n",
      "Epoch 00042: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3295 - accuracy: 0.8779 - val_loss: 0.4243 - val_accuracy: 0.8422\n",
      "Epoch 43/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3285 - accuracy: 0.8780\n",
      "Epoch 00043: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 29s 45ms/step - loss: 0.3288 - accuracy: 0.8779 - val_loss: 0.4321 - val_accuracy: 0.8398\n",
      "Epoch 44/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3279 - accuracy: 0.8791\n",
      "Epoch 00044: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 13s 20ms/step - loss: 0.3278 - accuracy: 0.8790 - val_loss: 0.4269 - val_accuracy: 0.8437\n",
      "Epoch 45/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3239 - accuracy: 0.8816\n",
      "Epoch 00045: val_accuracy did not improve from 0.84427\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3239 - accuracy: 0.8816 - val_loss: 0.4308 - val_accuracy: 0.8431\n",
      "Epoch 46/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3253 - accuracy: 0.8802\n",
      "Epoch 00046: val_accuracy improved from 0.84427 to 0.84558, saving model to best_model1.hdf5\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3253 - accuracy: 0.8802 - val_loss: 0.4284 - val_accuracy: 0.8456\n",
      "Epoch 47/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3253 - accuracy: 0.8802\n",
      "Epoch 00047: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3253 - accuracy: 0.8802 - val_loss: 0.4248 - val_accuracy: 0.8446\n",
      "Epoch 48/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3247 - accuracy: 0.8804\n",
      "Epoch 00048: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3247 - accuracy: 0.8804 - val_loss: 0.4351 - val_accuracy: 0.8419\n",
      "Epoch 49/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3215 - accuracy: 0.8833\n",
      "Epoch 00049: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3215 - accuracy: 0.8834 - val_loss: 0.4289 - val_accuracy: 0.8443\n",
      "Epoch 50/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3220 - accuracy: 0.8831\n",
      "Epoch 00050: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 10s 15ms/step - loss: 0.3220 - accuracy: 0.8831 - val_loss: 0.4282 - val_accuracy: 0.8451\n",
      "Epoch 51/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3212 - accuracy: 0.8806\n",
      "Epoch 00051: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3213 - accuracy: 0.8806 - val_loss: 0.4302 - val_accuracy: 0.8451\n",
      "Epoch 52/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3234 - accuracy: 0.8815\n",
      "Epoch 00052: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3233 - accuracy: 0.8815 - val_loss: 0.4270 - val_accuracy: 0.8443\n",
      "Epoch 53/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3162 - accuracy: 0.8836\n",
      "Epoch 00053: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3162 - accuracy: 0.8836 - val_loss: 0.4320 - val_accuracy: 0.8427\n",
      "Epoch 54/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3143 - accuracy: 0.8837\n",
      "Epoch 00054: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3141 - accuracy: 0.8838 - val_loss: 0.4349 - val_accuracy: 0.8408\n",
      "Epoch 55/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3189 - accuracy: 0.8822\n",
      "Epoch 00055: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 12ms/step - loss: 0.3188 - accuracy: 0.8822 - val_loss: 0.4332 - val_accuracy: 0.8441\n",
      "Epoch 56/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3151 - accuracy: 0.8845\n",
      "Epoch 00056: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3158 - accuracy: 0.8842 - val_loss: 0.4351 - val_accuracy: 0.8433\n",
      "Epoch 57/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3167 - accuracy: 0.8850\n",
      "Epoch 00057: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3169 - accuracy: 0.8849 - val_loss: 0.4372 - val_accuracy: 0.8437\n",
      "Epoch 58/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3119 - accuracy: 0.8854\n",
      "Epoch 00058: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 10s 15ms/step - loss: 0.3119 - accuracy: 0.8854 - val_loss: 0.4365 - val_accuracy: 0.8435\n",
      "Epoch 59/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3127 - accuracy: 0.8866\n",
      "Epoch 00059: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3124 - accuracy: 0.8868 - val_loss: 0.4283 - val_accuracy: 0.8444\n",
      "Epoch 60/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3165 - accuracy: 0.8849\n",
      "Epoch 00060: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 8s 13ms/step - loss: 0.3165 - accuracy: 0.8849 - val_loss: 0.4291 - val_accuracy: 0.8424\n",
      "Epoch 61/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3121 - accuracy: 0.8867\n",
      "Epoch 00061: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3123 - accuracy: 0.8867 - val_loss: 0.4323 - val_accuracy: 0.8449\n",
      "Epoch 62/70\n",
      "642/645 [============================>.] - ETA: 0s - loss: 0.3134 - accuracy: 0.8853\n",
      "Epoch 00062: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 13ms/step - loss: 0.3130 - accuracy: 0.8854 - val_loss: 0.4411 - val_accuracy: 0.8395\n",
      "Epoch 63/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3107 - accuracy: 0.8855\n",
      "Epoch 00063: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 13ms/step - loss: 0.3107 - accuracy: 0.8855 - val_loss: 0.4321 - val_accuracy: 0.8443\n",
      "Epoch 64/70\n",
      "644/645 [============================>.] - ETA: 0s - loss: 0.3109 - accuracy: 0.8869\n",
      "Epoch 00064: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3109 - accuracy: 0.8869 - val_loss: 0.4345 - val_accuracy: 0.8427\n",
      "Epoch 65/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3124 - accuracy: 0.8873\n",
      "Epoch 00065: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 11s 17ms/step - loss: 0.3119 - accuracy: 0.8875 - val_loss: 0.4353 - val_accuracy: 0.8421\n",
      "Epoch 66/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3062 - accuracy: 0.8881\n",
      "Epoch 00066: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 10s 15ms/step - loss: 0.3061 - accuracy: 0.8881 - val_loss: 0.4378 - val_accuracy: 0.8422\n",
      "Epoch 67/70\n",
      "643/645 [============================>.] - ETA: 0s - loss: 0.3090 - accuracy: 0.8863\n",
      "Epoch 00067: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3090 - accuracy: 0.8863 - val_loss: 0.4365 - val_accuracy: 0.8438\n",
      "Epoch 68/70\n",
      "641/645 [============================>.] - ETA: 0s - loss: 0.3067 - accuracy: 0.8876\n",
      "Epoch 00068: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3073 - accuracy: 0.8872 - val_loss: 0.4359 - val_accuracy: 0.8431\n",
      "Epoch 69/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3050 - accuracy: 0.8883\n",
      "Epoch 00069: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3050 - accuracy: 0.8883 - val_loss: 0.4407 - val_accuracy: 0.8405\n",
      "Epoch 70/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3090 - accuracy: 0.8881\n",
      "Epoch 00070: val_accuracy did not improve from 0.84558\n",
      "645/645 [==============================] - 9s 14ms/step - loss: 0.3090 - accuracy: 0.8881 - val_loss: 0.4393 - val_accuracy: 0.8390\n"
     ]
    }
   ],
   "source": [
    "model1 = Sequential()\n",
    "model1.add(layers.Embedding(max_words, 20))\n",
    "model1.add(layers.LSTM(15,dropout=0.5))\n",
    "model1.add(layers.Dense(3,activation='softmax'))\n",
    "\n",
    "\n",
    "model1.compile(optimizer='rmsprop',loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "#Implementing model checkpoins to save the best metric and do not lose it on training.\n",
    "checkpoint1 = ModelCheckpoint(\"best_model1.hdf5\", monitor='val_accuracy', verbose=1,save_best_only=True, mode='auto', period=1,save_weights_only=False)\n",
    "history = model1.fit(X_train, y_train, epochs=70,validation_data=(X_test, y_test),callbacks=[checkpoint1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 3.837682,
     "end_time": "2020-10-06T21:30:10.817414",
     "exception": false,
     "start_time": "2020-10-06T21:30:06.979732",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Bidirectional LTSM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:30:18.875895Z",
     "iopub.status.busy": "2020-10-06T21:30:18.874621Z",
     "iopub.status.idle": "2020-10-06T21:49:59.410271Z",
     "shell.execute_reply": "2020-10-06T21:49:59.411560Z"
    },
    "papermill": {
     "duration": 1184.636954,
     "end_time": "2020-10-06T21:49:59.411827",
     "exception": false,
     "start_time": "2020-10-06T21:30:14.774873",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.\n",
      "Epoch 1/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.7878 - accuracy: 0.6538\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.72901, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.7878 - accuracy: 0.6538 - val_loss: 0.6367 - val_accuracy: 0.7290\n",
      "Epoch 2/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.5628 - accuracy: 0.7749\n",
      "Epoch 00002: val_accuracy improved from 0.72901 to 0.77805, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 54s 83ms/step - loss: 0.5628 - accuracy: 0.7749 - val_loss: 0.5308 - val_accuracy: 0.7781\n",
      "Epoch 3/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4970 - accuracy: 0.8078\n",
      "Epoch 00003: val_accuracy improved from 0.77805 to 0.80949, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 55s 85ms/step - loss: 0.4970 - accuracy: 0.8078 - val_loss: 0.4920 - val_accuracy: 0.8095\n",
      "Epoch 4/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4665 - accuracy: 0.8208\n",
      "Epoch 00004: val_accuracy improved from 0.80949 to 0.81517, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 61s 95ms/step - loss: 0.4665 - accuracy: 0.8208 - val_loss: 0.4771 - val_accuracy: 0.8152\n",
      "Epoch 5/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4487 - accuracy: 0.8275\n",
      "Epoch 00005: val_accuracy improved from 0.81517 to 0.81706, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 57s 89ms/step - loss: 0.4487 - accuracy: 0.8275 - val_loss: 0.4657 - val_accuracy: 0.8171\n",
      "Epoch 6/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4343 - accuracy: 0.8354\n",
      "Epoch 00006: val_accuracy improved from 0.81706 to 0.82215, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 58s 89ms/step - loss: 0.4343 - accuracy: 0.8354 - val_loss: 0.4589 - val_accuracy: 0.8222\n",
      "Epoch 7/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4192 - accuracy: 0.8421\n",
      "Epoch 00007: val_accuracy improved from 0.82215 to 0.82332, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.4192 - accuracy: 0.8421 - val_loss: 0.4535 - val_accuracy: 0.8233\n",
      "Epoch 8/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4090 - accuracy: 0.8457\n",
      "Epoch 00008: val_accuracy did not improve from 0.82332\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.4090 - accuracy: 0.8457 - val_loss: 0.4588 - val_accuracy: 0.8133\n",
      "Epoch 9/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.4038 - accuracy: 0.8494\n",
      "Epoch 00009: val_accuracy improved from 0.82332 to 0.82637, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.4038 - accuracy: 0.8494 - val_loss: 0.4423 - val_accuracy: 0.8264\n",
      "Epoch 10/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3960 - accuracy: 0.8510\n",
      "Epoch 00010: val_accuracy improved from 0.82637 to 0.83190, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 54s 84ms/step - loss: 0.3960 - accuracy: 0.8510 - val_loss: 0.4435 - val_accuracy: 0.8319\n",
      "Epoch 11/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3914 - accuracy: 0.8553\n",
      "Epoch 00011: val_accuracy did not improve from 0.83190\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3914 - accuracy: 0.8553 - val_loss: 0.4416 - val_accuracy: 0.8310\n",
      "Epoch 12/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3852 - accuracy: 0.8565\n",
      "Epoch 00012: val_accuracy improved from 0.83190 to 0.83496, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3852 - accuracy: 0.8565 - val_loss: 0.4301 - val_accuracy: 0.8350\n",
      "Epoch 13/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3778 - accuracy: 0.8573\n",
      "Epoch 00013: val_accuracy improved from 0.83496 to 0.83554, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3778 - accuracy: 0.8573 - val_loss: 0.4308 - val_accuracy: 0.8355\n",
      "Epoch 14/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3775 - accuracy: 0.8592\n",
      "Epoch 00014: val_accuracy did not improve from 0.83554\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3775 - accuracy: 0.8592 - val_loss: 0.4324 - val_accuracy: 0.8338\n",
      "Epoch 15/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3683 - accuracy: 0.8642\n",
      "Epoch 00015: val_accuracy did not improve from 0.83554\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3683 - accuracy: 0.8642 - val_loss: 0.4337 - val_accuracy: 0.8339\n",
      "Epoch 16/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3651 - accuracy: 0.8658\n",
      "Epoch 00016: val_accuracy improved from 0.83554 to 0.83801, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3651 - accuracy: 0.8658 - val_loss: 0.4333 - val_accuracy: 0.8380\n",
      "Epoch 17/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3607 - accuracy: 0.8664\n",
      "Epoch 00017: val_accuracy improved from 0.83801 to 0.83831, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3607 - accuracy: 0.8664 - val_loss: 0.4263 - val_accuracy: 0.8383\n",
      "Epoch 18/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3573 - accuracy: 0.8682\n",
      "Epoch 00018: val_accuracy did not improve from 0.83831\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3573 - accuracy: 0.8682 - val_loss: 0.4283 - val_accuracy: 0.8367\n",
      "Epoch 19/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3562 - accuracy: 0.8688\n",
      "Epoch 00019: val_accuracy did not improve from 0.83831\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3562 - accuracy: 0.8688 - val_loss: 0.4274 - val_accuracy: 0.8380\n",
      "Epoch 20/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3523 - accuracy: 0.8702\n",
      "Epoch 00020: val_accuracy improved from 0.83831 to 0.84034, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3523 - accuracy: 0.8702 - val_loss: 0.4204 - val_accuracy: 0.8403\n",
      "Epoch 21/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3505 - accuracy: 0.8717\n",
      "Epoch 00021: val_accuracy did not improve from 0.84034\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3505 - accuracy: 0.8717 - val_loss: 0.4322 - val_accuracy: 0.8393\n",
      "Epoch 22/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3474 - accuracy: 0.8709\n",
      "Epoch 00022: val_accuracy did not improve from 0.84034\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3474 - accuracy: 0.8709 - val_loss: 0.4316 - val_accuracy: 0.8380\n",
      "Epoch 23/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3460 - accuracy: 0.8737\n",
      "Epoch 00023: val_accuracy improved from 0.84034 to 0.84296, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3460 - accuracy: 0.8737 - val_loss: 0.4274 - val_accuracy: 0.8430\n",
      "Epoch 24/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3424 - accuracy: 0.8732\n",
      "Epoch 00024: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.3424 - accuracy: 0.8732 - val_loss: 0.4258 - val_accuracy: 0.8422\n",
      "Epoch 25/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3401 - accuracy: 0.8750\n",
      "Epoch 00025: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 51s 80ms/step - loss: 0.3401 - accuracy: 0.8750 - val_loss: 0.4330 - val_accuracy: 0.8409\n",
      "Epoch 26/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3408 - accuracy: 0.8752\n",
      "Epoch 00026: val_accuracy did not improve from 0.84296\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.3408 - accuracy: 0.8752 - val_loss: 0.4289 - val_accuracy: 0.8382\n",
      "Epoch 27/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3349 - accuracy: 0.8784\n",
      "Epoch 00027: val_accuracy improved from 0.84296 to 0.84369, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.3349 - accuracy: 0.8784 - val_loss: 0.4244 - val_accuracy: 0.8437\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3327 - accuracy: 0.8773\n",
      "Epoch 00028: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 51s 80ms/step - loss: 0.3327 - accuracy: 0.8773 - val_loss: 0.4298 - val_accuracy: 0.8390\n",
      "Epoch 29/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3324 - accuracy: 0.8771\n",
      "Epoch 00029: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 51s 80ms/step - loss: 0.3324 - accuracy: 0.8771 - val_loss: 0.4278 - val_accuracy: 0.8435\n",
      "Epoch 30/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3306 - accuracy: 0.8789\n",
      "Epoch 00030: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.3306 - accuracy: 0.8789 - val_loss: 0.4275 - val_accuracy: 0.8406\n",
      "Epoch 31/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3275 - accuracy: 0.8811\n",
      "Epoch 00031: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 81ms/step - loss: 0.3275 - accuracy: 0.8811 - val_loss: 0.4287 - val_accuracy: 0.8431\n",
      "Epoch 32/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3244 - accuracy: 0.8799\n",
      "Epoch 00032: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 51s 79ms/step - loss: 0.3244 - accuracy: 0.8799 - val_loss: 0.4312 - val_accuracy: 0.8409\n",
      "Epoch 33/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3222 - accuracy: 0.8821\n",
      "Epoch 00033: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 55s 85ms/step - loss: 0.3222 - accuracy: 0.8821 - val_loss: 0.4278 - val_accuracy: 0.8399\n",
      "Epoch 34/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3195 - accuracy: 0.8839\n",
      "Epoch 00034: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3195 - accuracy: 0.8839 - val_loss: 0.4311 - val_accuracy: 0.8406\n",
      "Epoch 35/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3207 - accuracy: 0.8810\n",
      "Epoch 00035: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3207 - accuracy: 0.8810 - val_loss: 0.4256 - val_accuracy: 0.8417\n",
      "Epoch 36/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3181 - accuracy: 0.8839\n",
      "Epoch 00036: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3181 - accuracy: 0.8839 - val_loss: 0.4351 - val_accuracy: 0.8409\n",
      "Epoch 37/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3153 - accuracy: 0.8854\n",
      "Epoch 00037: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3153 - accuracy: 0.8854 - val_loss: 0.4313 - val_accuracy: 0.8405\n",
      "Epoch 38/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3134 - accuracy: 0.8847\n",
      "Epoch 00038: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3134 - accuracy: 0.8847 - val_loss: 0.4327 - val_accuracy: 0.8419\n",
      "Epoch 39/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3158 - accuracy: 0.8839\n",
      "Epoch 00039: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3158 - accuracy: 0.8839 - val_loss: 0.4301 - val_accuracy: 0.8396\n",
      "Epoch 40/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3131 - accuracy: 0.8849\n",
      "Epoch 00040: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3131 - accuracy: 0.8849 - val_loss: 0.4325 - val_accuracy: 0.8409\n",
      "Epoch 41/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3132 - accuracy: 0.8845\n",
      "Epoch 00041: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3132 - accuracy: 0.8845 - val_loss: 0.4363 - val_accuracy: 0.8395\n",
      "Epoch 42/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3079 - accuracy: 0.8876\n",
      "Epoch 00042: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3079 - accuracy: 0.8876 - val_loss: 0.4380 - val_accuracy: 0.8411\n",
      "Epoch 43/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3078 - accuracy: 0.8884\n",
      "Epoch 00043: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3078 - accuracy: 0.8884 - val_loss: 0.4331 - val_accuracy: 0.8418\n",
      "Epoch 44/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3042 - accuracy: 0.8890\n",
      "Epoch 00044: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 54s 84ms/step - loss: 0.3042 - accuracy: 0.8890 - val_loss: 0.4363 - val_accuracy: 0.8373\n",
      "Epoch 45/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3045 - accuracy: 0.8885\n",
      "Epoch 00045: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3045 - accuracy: 0.8885 - val_loss: 0.4324 - val_accuracy: 0.8412\n",
      "Epoch 46/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3022 - accuracy: 0.8903\n",
      "Epoch 00046: val_accuracy did not improve from 0.84369\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.3022 - accuracy: 0.8903 - val_loss: 0.4388 - val_accuracy: 0.8412\n",
      "Epoch 47/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3017 - accuracy: 0.8893\n",
      "Epoch 00047: val_accuracy improved from 0.84369 to 0.84544, saving model to best_model2.hdf5\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3017 - accuracy: 0.8893 - val_loss: 0.4360 - val_accuracy: 0.8454\n",
      "Epoch 48/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3006 - accuracy: 0.8890\n",
      "Epoch 00048: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3006 - accuracy: 0.8890 - val_loss: 0.4393 - val_accuracy: 0.8427\n",
      "Epoch 49/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2963 - accuracy: 0.8916\n",
      "Epoch 00049: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.2963 - accuracy: 0.8916 - val_loss: 0.4402 - val_accuracy: 0.8419\n",
      "Epoch 50/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.3002 - accuracy: 0.8905\n",
      "Epoch 00050: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.3002 - accuracy: 0.8905 - val_loss: 0.4366 - val_accuracy: 0.8403\n",
      "Epoch 51/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2979 - accuracy: 0.8909\n",
      "Epoch 00051: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 51s 80ms/step - loss: 0.2979 - accuracy: 0.8909 - val_loss: 0.4435 - val_accuracy: 0.8414\n",
      "Epoch 52/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2963 - accuracy: 0.8931\n",
      "Epoch 00052: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.2963 - accuracy: 0.8931 - val_loss: 0.4406 - val_accuracy: 0.8408\n",
      "Epoch 53/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2905 - accuracy: 0.8958\n",
      "Epoch 00053: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 51s 80ms/step - loss: 0.2905 - accuracy: 0.8958 - val_loss: 0.4484 - val_accuracy: 0.8434\n",
      "Epoch 54/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2937 - accuracy: 0.8932\n",
      "Epoch 00054: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2937 - accuracy: 0.8932 - val_loss: 0.4364 - val_accuracy: 0.8414\n",
      "Epoch 55/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2947 - accuracy: 0.8959\n",
      "Epoch 00055: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 51s 79ms/step - loss: 0.2947 - accuracy: 0.8959 - val_loss: 0.4426 - val_accuracy: 0.8428\n",
      "Epoch 56/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2938 - accuracy: 0.8932\n",
      "Epoch 00056: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 54s 83ms/step - loss: 0.2938 - accuracy: 0.8932 - val_loss: 0.4373 - val_accuracy: 0.8408\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2896 - accuracy: 0.8946\n",
      "Epoch 00057: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.2896 - accuracy: 0.8946 - val_loss: 0.4396 - val_accuracy: 0.8431\n",
      "Epoch 58/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2957 - accuracy: 0.8930\n",
      "Epoch 00058: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2957 - accuracy: 0.8930 - val_loss: 0.4371 - val_accuracy: 0.8392\n",
      "Epoch 59/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2905 - accuracy: 0.8957\n",
      "Epoch 00059: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2905 - accuracy: 0.8957 - val_loss: 0.4446 - val_accuracy: 0.8418\n",
      "Epoch 60/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2868 - accuracy: 0.8961\n",
      "Epoch 00060: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2868 - accuracy: 0.8961 - val_loss: 0.4475 - val_accuracy: 0.8411\n",
      "Epoch 61/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2888 - accuracy: 0.8960\n",
      "Epoch 00061: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.2888 - accuracy: 0.8960 - val_loss: 0.4446 - val_accuracy: 0.8409\n",
      "Epoch 62/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2900 - accuracy: 0.8940\n",
      "Epoch 00062: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2900 - accuracy: 0.8940 - val_loss: 0.4467 - val_accuracy: 0.8418\n",
      "Epoch 63/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2889 - accuracy: 0.8966\n",
      "Epoch 00063: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.2889 - accuracy: 0.8966 - val_loss: 0.4472 - val_accuracy: 0.8405\n",
      "Epoch 64/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2817 - accuracy: 0.8983\n",
      "Epoch 00064: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 80ms/step - loss: 0.2817 - accuracy: 0.8983 - val_loss: 0.4404 - val_accuracy: 0.8431\n",
      "Epoch 65/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2843 - accuracy: 0.8976\n",
      "Epoch 00065: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.2843 - accuracy: 0.8976 - val_loss: 0.4511 - val_accuracy: 0.8435\n",
      "Epoch 66/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2857 - accuracy: 0.8977\n",
      "Epoch 00066: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2857 - accuracy: 0.8977 - val_loss: 0.4410 - val_accuracy: 0.8438\n",
      "Epoch 67/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2840 - accuracy: 0.8971\n",
      "Epoch 00067: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 82ms/step - loss: 0.2840 - accuracy: 0.8971 - val_loss: 0.4433 - val_accuracy: 0.8437\n",
      "Epoch 68/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2829 - accuracy: 0.8983\n",
      "Epoch 00068: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 53s 83ms/step - loss: 0.2829 - accuracy: 0.8983 - val_loss: 0.4445 - val_accuracy: 0.8412\n",
      "Epoch 69/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2856 - accuracy: 0.8967\n",
      "Epoch 00069: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2856 - accuracy: 0.8967 - val_loss: 0.4387 - val_accuracy: 0.8438\n",
      "Epoch 70/70\n",
      "645/645 [==============================] - ETA: 0s - loss: 0.2813 - accuracy: 0.8988\n",
      "Epoch 00070: val_accuracy did not improve from 0.84544\n",
      "645/645 [==============================] - 52s 81ms/step - loss: 0.2813 - accuracy: 0.8988 - val_loss: 0.4455 - val_accuracy: 0.8433\n"
     ]
    }
   ],
   "source": [
    "model2 = Sequential()\n",
    "model2.add(layers.Embedding(max_words, 40, input_length=max_len))\n",
    "model2.add(layers.Bidirectional(layers.LSTM(20,dropout=0.6)))\n",
    "model2.add(layers.Dense(3,activation='softmax'))\n",
    "model2.compile(optimizer='rmsprop',loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "#Implementing model checkpoins to save the best metric and do not lose it on training.\n",
    "checkpoint2 = ModelCheckpoint(\"best_model2.hdf5\", monitor='val_accuracy', verbose=1,save_best_only=True, mode='auto', period=1,save_weights_only=False)\n",
    "history = model2.fit(X_train, y_train, epochs=70,validation_data=(X_test, y_test),callbacks=[checkpoint2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 13.181114,
     "end_time": "2020-10-06T21:57:15.037099",
     "exception": false,
     "start_time": "2020-10-06T21:57:01.855985",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "If you check the val_accuracy metric in the training logs you won't find better score than the one achieved by the BidRNN. Again, the previous model is not the best for this task becaue its majorly used for short translation tasks, but the good thing to notice is its speed to train.\n",
    "\n",
    "Let's move on."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 13.897718,
     "end_time": "2020-10-06T21:57:43.244943",
     "exception": false,
     "start_time": "2020-10-06T21:57:29.347225",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Best model validation\n",
    "(Before final commit, the best model obtained was the BidRNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:58:11.467383Z",
     "iopub.status.busy": "2020-10-06T21:58:11.466210Z",
     "iopub.status.idle": "2020-10-06T21:58:12.123269Z",
     "shell.execute_reply": "2020-10-06T21:58:12.122665Z"
    },
    "papermill": {
     "duration": 14.579894,
     "end_time": "2020-10-06T21:58:12.123401",
     "exception": false,
     "start_time": "2020-10-06T21:57:57.543507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Let's load the best model obtained during training\n",
    "best_model = keras.models.load_model(\"best_model2.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:58:39.201557Z",
     "iopub.status.busy": "2020-10-06T21:58:39.200429Z",
     "iopub.status.idle": "2020-10-06T21:58:42.153698Z",
     "shell.execute_reply": "2020-10-06T21:58:42.152736Z"
    },
    "papermill": {
     "duration": 16.163436,
     "end_time": "2020-10-06T21:58:42.153869",
     "exception": false,
     "start_time": "2020-10-06T21:58:25.990433",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 - 3s - loss: 0.4360 - accuracy: 0.8454\n",
      "Model accuracy:  0.8454373478889465\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = best_model.evaluate(X_test, y_test, verbose=2)\n",
    "print('Model accuracy: ',test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T21:59:08.598704Z",
     "iopub.status.busy": "2020-10-06T21:59:08.597660Z",
     "iopub.status.idle": "2020-10-06T21:59:11.143379Z",
     "shell.execute_reply": "2020-10-06T21:59:11.144225Z"
    },
    "papermill": {
     "duration": 15.873978,
     "end_time": "2020-10-06T21:59:11.144400",
     "exception": false,
     "start_time": "2020-10-06T21:58:55.270422",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "predictions = best_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 13.54467,
     "end_time": "2020-10-06T21:59:38.859073",
     "exception": false,
     "start_time": "2020-10-06T21:59:25.314403",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Confusion matrix\n",
    "\n",
    "Alright, we all know the accuracy is not a good metric to measure how well a model is. That's the reason why I like to always see its confusion matrix, that way I have a better understanding of its classification and generalization ability. Let's plot it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:00:05.244627Z",
     "iopub.status.busy": "2020-10-06T22:00:05.243340Z",
     "iopub.status.idle": "2020-10-06T22:00:05.261173Z",
     "shell.execute_reply": "2020-10-06T22:00:05.260523Z"
    },
    "papermill": {
     "duration": 13.356845,
     "end_time": "2020-10-06T22:00:05.261290",
     "exception": false,
     "start_time": "2020-10-06T21:59:51.904445",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "matrix = confusion_matrix(y_test.argmax(axis=1), np.around(predictions, decimals=0).argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:00:32.586908Z",
     "iopub.status.busy": "2020-10-06T22:00:32.586210Z",
     "iopub.status.idle": "2020-10-06T22:00:33.285612Z",
     "shell.execute_reply": "2020-10-06T22:00:33.285009Z"
    },
    "papermill": {
     "duration": 14.015558,
     "end_time": "2020-10-06T22:00:33.285736",
     "exception": false,
     "start_time": "2020-10-06T22:00:19.270178",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fd37911d510>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyYAAANOCAYAAADkkBgFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd5idVbk34N9KAgRIoxxAASVCQPCggIgFRAWpKkgRBP1QhBNRENSjUiygWLCA5VAjoGBDQZqCgAVpohC6lEAAgRhAEJKQhJbM+v5IGCfJzmQCmf1OMvd9XfvKftde79prRzeZZ55nrVVqrQEAAGjSgKYnAAAAIDABAAAaJzABAAAaJzABAAAaJzABAAAaN6i33+D5x++z7Rc0YJW1tm16CtAvPdcxo+kpQL81bfo/StNz6Im+/vPxUiu/qpG/RxkTAACgcQITAACgcQITAACgcb2+xgQAAOiiY2bTM3jJSinbJ/l+koFJTq21HjPX6yskOT3J2kmeSfKRWuvfuxtTxgQAAOixUsrAJCck2SHJBkn2KqVsMFe3I5LcXGt9bZJ9MiuI6ZbABAAAWBibJRlfa72v1vpckrOS7DxXnw2S/DFJaq13JVmrlLJqd4Mq5QIAgHaqHU3P4KVaPclDXa4nJHnjXH1uSbJrkqtLKZsleWWSNZI8Or9BZUwAAIBOpZTRpZSxXR6j5+7S4ra5z2Y5JskKpZSbk3wiyU1Juj3oScYEAADoVGsdk2RMN10mJFmzy/UaSSbONcaUJPsmSSmlJLl/9mO+BCYAANBOHYt9Kdf1SUaVUkYm+WeS9yfZu2uHUsqIJNNnr0HZP8mVs4OV+RKYAAAAPVZrnVFKOSjJpZm1XfDptdbbSykHzH795CTrJzmzlDIzyR1J9lvQuAITAABgodRaL05y8VxtJ3d5fm2SUQszpsXvAABA42RMAACgjeriv11wr5AxAQAAGicwAQAAGqeUCwAA2mnx3y64V8iYAAAAjROYAAAAjVPKBQAA7WRXrpZkTAAAgMYJTAAAgMYp5QIAgHbqmNn0DPokGRMAAKBxAhMAAKBxSrkAAKCd7MrVkowJAADQOIEJAADQOIEJAADQOGtMAACgnTqsMWlFxgQAAGicwAQAAGicUi4AAGijarvglmRMAACAxglMAACAxinlAgCAdrIrV0syJgAAQOMEJgAAQOOUcgEAQDvZlaslGRMAAKBxAhMAAKBxSrkAAKCdOmY2PYM+ScYEAABonMAEAABonFIuAABoJ7tytSRjAgAANE5gAgAANE5gAgAANM4aEwAAaKcOa0xakTEBAAAaJzABAAAap5QLAADayXbBLcmYAAAAjROYAAAAjVPKBQAA7WRXrpZkTAAAgMYJTAAAgMYp5QIAgDaqdWbTU+iTZEwAAIDGCUwAAIDGKeUCAIB2csBiSzImAABA4wQmAABA4wQmAABA46wxAQCAdnLye0syJgAAQOMEJgAAQOOUcgEAQDvZLrglGRMAAKBxAhMAAKBxSrkAAKCdOmY2PYM+ScYEAABonMAEAABonFIuAABoJ7tytSRjAgAANE5gAgAANE4pFwAAtFOHUq5WZEwAAIDGCUwAAIDGCUwAAIDGWWMCAADtZLvglmRMAACAxglMAACAxinlAgCAdrJdcEsyJgAAQOMEJgAAQOOUcgEAQDsp5WpJxgQAAGicwAQAAGicUi4AAGijWmc2PYU+ScYEAABonMAEAABonFIuAABoJ7tytSRjAgAANE5gAgAANE5gAgAA7VQ7+vajB0op25dSxpVSxpdSDmvx+vBSym9KKbeUUm4vpey7oDEFJgAAQI+VUgYmOSHJDkk2SLJXKWWDubodmOSOWuvrkrw9ybGllKW7G1dgAgAALIzNkoyvtd5Xa30uyVlJdp6rT00ytJRSkgxJ8kSSGd0NKjABAAA6lVJGl1LGdnmMnqvL6kke6nI9YXZbV8cnWT/JxCS3JTmk1u7rxGwXDAAA7dTHtwuutY5JMqabLqXVbXNdb5fk5iRbJVk7ye9LKVfVWqfMb1AZEwAAYGFMSLJml+s1Misz0tW+Sc6ts4xPcn+SV3c3qMAEAABYGNcnGVVKGTl7Qfv7k1w4V58Hk2ydJKWUVZOsl+S+7gZVygUAAO3Uwy15+6pa64xSykFJLk0yMMnptdbbSykHzH795CRHJ/lxKeW2zCr9OrTW+nh34wpMAACAhVJrvTjJxXO1ndzl+cQk2y7MmEq5AACAxsmYAABAO/XxXbmaImMCAAA0TmACAAA0TikXAAC002K+K1dvkTEBAAAaJzABAAAap5QLAADaya5cLcmYAAAAjROYAAAAjROYAAAAjbPGBAAA2skak5ZkTAAAgMYJTAAAgMYp5QIAgHZy8ntLMiYAAEDjBCYAAEDjlHIBAEA72ZWrJRkTAACgcQITAACgcUq5AACgnezK1ZKMCQAA0DiBCQAA0DilXAAA0E525WpJxgQAAGicwKQfuPf+B7LfwYdl063em3fs9IEc/8MzM3PmzAXeN/6+B/I/nzwim2713myx4575yrf/L9OnPz1Hn79cd2M+e+Qx2Xa3D+W/N98hJ5z20976GNDnrffqdXL+b8/MP/91W+6455oc/oVDMmDAgv8zO2zYkBx/0jG5/6Eb8sA/b8qY047NCiuOmKPPCSd/M09OHT/PY9S6r+rs8+r1R+Xs807PHfdck0f+fUduu/PKfP/4r2fVVf9rkX9W6Ete/ep1ctFFP8tjj9+Z8ff+LV/44qd6+N0bmpNP+XYm/POWTHz41px++vey4lzfvc9/4VO57rpL8vAjt+WRR/+eq66+MLvt9u55xlp//VG58MIz89jjd+aBB2/M977/1Sy//HKL7DNCf6CUawk3ecpT2f+QI7L2yFfkB8d8KQ/98+F85/gfpqPWHDz6Q/O976mp0/KRgw/LWmuunu985bBMmvxUjjvxtDz+7yfzg2O+1Nnvmr/dkLvH3583brpRLvnDFe34SNAnDR8xLOf95oyMu2t8PrDnARn5qlfk6K8fngEDBuRrX/lut/eedsYPMmrUyBx80BHp6OjIUV/5XH521knZcdu95ug3btz4HHTAYXO0PfjAhM7nw4YNyYMPPJRf/vy8PPzIv/LKV66RQw//RDba+L+z1Za79OgXErC4GTFiWH570c9y1533ZM89/icjX/XKfOMbn8+AAQPylS8f2+29Z/7k+Iwa9aoc+PFD01Frjj760Jz1yzHZdps9OvsMGzokP/3pObnrrvGZOXNm3rvLDjnzJ8dn5syZOf/8383qM2xoLr7457ln/P3ZZ5+DstKKK+SrXz0sq622St6/5+he/fwspuzK1ZLAZAn3q/MvzrPPPZfvff0LGbL88kmSadOn58TTfpaPfGD3zra5nXXub/Pss8/m+G8dlWFDhyRJhg8bmk8c9uX8/c6789/rr5sk+d8D98tnP/E/SZLLr/prGz4R9E0f2W/vLDt4cPbZ+8A89dTU/PnyazJ06JAcesTB+cF3f5innpra8r43bLZx3rnNlnnXdnvlL9dcnyR5eOKj+eMV5+Ztb39LrvjzXzr7Tp/2dMZef/N853Dd327KdX+7qfP6mqv+lon/fCTn/eaMvOa/X51bb7l9EX1a6Dv23/+DGTx4cPba64BZ37M/XZ1hQ4fkiM9/Mt897pT5fvc222yTbLPN27LtNnvkmmuuS5JMnPhIrrzygrzjHZvn8suvSZIceujRc9z3xz9elfXXXzd7f2DXzsBk9Oj/l8HLDs77dt8/kydPSZI88eSknH32qdl4kw1z04239dbHhyWKUq4l3NV/HZu3bLbJHAHIDlu/Lc88+2zG3jT//1Dedc99ec2r1+0MSpLkLZttklJKrrz2+s62nqTKoT9457Zb5k9/vGqOH4LOPee3WW65ZbP5Fpt1e9+jjz7WGZQkyY033Jp/3P9g3rnt217yvJ54YlKSZOmll3rJY0FftM22b8sf/nDlHN+9s8/+TZZbbtls8dY3zve+bbd7Wx599LHOoCRJbhh7S+6//8Fsu+3bu33PJ554MksvvXTn9Yav3SA33XhbZ1CSJH/8w1Xp6OjI9ttv9SI+FfRPfqpcwt3/wEMZ+co152h72WqrZNnBy+S+LiUgc3vuueey1FJzJtQGDhyYAQNK7vvHg70yV1icjVp37dx9931ztE2Y8HCmTZs+xzqQVvfdM9d9SXL3uHuz7lz3rffqdfLAxJvzyL/vyO8uOytvmU/AU0rJUkstlXVGjcyRX/lsbhh7S24Ye8uL+FTQ96237tq5++5752ibMGFipk2bnvXWXXu+96277toZN+7eedrHjRufddeb976BAwdm+PBh2XPPnbP11lvm1FN/1vna4MHL5Lnnn5+j/4wZM9LR0ZH11ltnYT8S9FvdlnKVUlbs7vVa6xOLdjosalOempphQ+Yt1xo2dEimzCe9nSSvWOPluej3l+f5GTOy1KBZ/ze5Y9w9mTmzI5OnPNVr84XF1YgRwzJ50pR52idNmpIRI4a/qPvWWus/v1S49dY7csPYW3LXXeOz8sor5sBP7JfzLvxxdtjm/bnxhlvnuPdX556Wd26zZZLkphtvyx677Zda64v9aNCnjVhh+Hy+Q5MzYoX5f/dWGDF8jgxH531PTs5aI18xR9sb3rBx/nzFeUmS559/Pp/+9JH57W8u63z9vnv/kT323DmDBg3KjBkzkiQbb7JhBg0alBW7mQP9mO2CW1pQxuSGJGNn/zn3Y+z8biqljC6ljC2ljD31zF8sqrnyYpUyT1OtLZs77bbT9nly0uR8/biT8vi/n8j4+x7IV489IQMHDsjAgQN7cbKw+KqZ94f/Ulq39+i+LsHEKSeekdNP/Xn+cvV1ufD8S7Lzuz6Yhyc+mk9/5mPz3HvoZ76cd759t3x0v//N8kOWy9nnnp5llll6nn6wpGgVeJdSFhiQ9/S+22+/K1ts8Z68+10fyCknn5njjvty3ve+nTpf/9GPzsrKK6+YY4/7clZd9b+y/vqj8r3vHZ0ZM2Zkph9Aoce6zZjUWke+mEFrrWOSjEmS5x+/z6/pGjRs6JA8NXXaPO1PTZuWoUOGtLhjlle9cs0c+bmD860fjMnZF1ycAQMGZPedtk9SstIKI+Z7H/RXkyZNyfDhw+ZpHzZsaMvf5na9b+WV501ODx8+rOVvc1/wzDPP5veXXZHtd5i3fv2+ex9I8kBuGHtLrv3L9bn59j9n9z12ys9+ck7PPgwsRiY9OTnDRyz8d+/JSZOz8sorzdM+fMS8373p05/uXMB++eXXZNjwoTn6q4fm7LMvTJLcffe9+cRBh+eYb34x++//gcycOTOnn/6L1Frzr0cffykfD/qVHu/KVUpZIcmoJINfaKu1Xtkbk2LRGfnKNXP/Aw/N0fbwo4/l6aefyateuUa39+767u3yrm3ekQcm/DMrrjAiKwwfli123DO7vWe73pwyLJbuuXveNSGrr/6yDBmyfMs1JF3ve/NbNp2nfdS6a+ei3/5+ge+7oN8IP/TQxDz55KSsNXLNbvvB4mrc3ffOs5bkhe/euLvnXUPygrvvvjebbz7vOq111117jjKtVm6++e/ZZ5895ijdOvPMs/PLX16YddZZK4899u88/vgTeWjCzfnxj3/5Ij4VSzyZtJZ6tPi9lLJ/kiuTXJrky7P/PKr3psWissWbNs01f7sh06ZN72y75I9XZPAyy2TTjTdc4P3LLLN01l17ZFZecYX89tI/zdphZOste3PKsFj6w2VXZqut35ohXdZ07bLbjpk+/elcc/V13d632mqr5E1vfn1n20Yb/3dGvuoV+cNl8z8baPDgZfLObbbMzTf/vdt5rTNqZFZaacU88I/5b3YBi7PfX3ZFtn7nlnN893bf/d2ZPv3pXH3V3+Z732WXXpHVVlslb37zf34xsPEmG+ZVr3plLrvsz92+55vftGkmTJjYGZS84Nlnn83tt4/Lv/71ePbaa5cMGFBy7q9/++I+GPRDPc2YHJLkDUn+Wmt9Rynl1ZkVoNDH7fHeHfOzcy7IIUd8Nft98H2ZMPHhnHj6z7LP+3eZcwvhPT6STTfeMEcf/qkkydRp0zLmjLPy+o02zKCBA3PdjbfkjF+cm6MOPSTDhw3tvG/iI4/m73fenWTWgsD7/vFgLrv8qiw7eHDe+uY3tPfDQoNOP+3nGf2xfXLmz0/I948bk7VGrplDjzg4Jx5/+hzbmN5wyx9zzdXX5eADD0+SXH/dTfnD76/MSWO+nS9+/pjOAxav/cv1nWeYDBs2JGed88P86qwLct99D2SllVbIxw78SF728tWy7z4Hd479la8dlpkzZmbs2JszefJTWW+9tXPwJ/8n9937QM49xw9HLJlOPfWn+djHP5xf/OLkHHfcyVlr5CtyxOc/mf/7v1Pn+O7detufc/XVf8vHP3ZokuS6627M739/RX546nE54oivpaNj1gGL11xzXecZJmuuuXpOOeXb+eWvLsg/7n8wyw9ZPjvttF3et8dOOfjgz3eOPXTokHzucwfl6mv+lpkzZmbLLd+cgw/ZPwcdeHiefHJye/9CYDHW08DkmVrrM6WUlFKWqbXeVUpZr1dnxiIxfNjQnPb9b+Rrx52Ugz53VIYOXT777LFLPr7fB+boN3PmzHTM/E9accCAgbnz7ntzzoWX5Nlnn8s6r3pljv3qEdl6y7fMcd91N9yaL3z9uM7rS/90VS7901V5+Wqr5LJfn9G7Hw76kMmTpuS9794n3zr2yPzi7DGZPHlKTjrhRznmaz+Yo9+gQQMzcOCcyer9PnxIvn7M53P8icekDBiQyy75Uw797H8OdXv22efy+ONP5DOfOzAr/9dKefaZZ3PddTfl3dvvnZtv+k/G5Oabbsvoj+6TD+27Z5YZvEwmTJiYCy+4NN899uRMn/507/4FQEMmTZqSd+24d4477is5+5zTMnnylBx//Gn52le/N0e/QYMGZeCAOTdv+dA+n8g3v/XFnHTStzNgQMklv/tTPvOZozpfnzx5Sh5++NEceugnsuqq/5XJk6fkrjvvya67fDiXXvrnzn4zZ87M6163QT687/uz7LKDc8cd4/LBDx64wJIw+jE7JbZUerKFZCnlvCT7Jvlkkq2SPJlkqVrrjgu61+J3aMYqa23b9BSgX3quY8aCOwG9Ytr0f3Sz52jf8fQvv9ynfz5eds8jG/l77FHGpNa6y+ynR5VSLk8yPMklvTYrAACgX1lgYFJKGZDk1lrrfydJrXX+qzEBAIDu2ZWrpQXuylVr7UhySynlFQvqCwAA8GL0dPH7y5LcXkq5LknnaX211p3mfwsAAEDP9DQwsTUwAAAsCkq5WuppYLJjrfXQrg2llG8msd4EAAB4yXp08nuSbVq07bAoJwIAAPRf3WZMSikfS/LxJGuXUm7t8tLQJH/pzYkBAAD9x4JKuX6e5HdJvpHksC7tT9Van+i1WQEAwJKqWmPSSreBSa11cpLJpZRD53ppSCllSK31wd6bGgAA0F/0dPH7RUlqkpJkcJKRScYleU0vzQsAAOhHehSY1Fo37HpdStkkyUd7ZUYAALAks11wSz3dlWsOtdYbk7xhEc8FAADop3qUMSmlfLrL5YAkmyR5rFdmBAAA9Ds9XWMytMvzGZm15uTXi346AACwhKu16Rn0ST1dY/LlJCmlLF9rnda7UwIAAPqbHq0xKaW8uZRyR5I7Z1+/rpRyYq/ODAAA6Dd6Wsr1vSTbJbkwSWqtt5RStuy1WQEAwJLKrlwt9XhXrlrrQ3M1zVzEcwEAAPqpnmZMHiqlvCVJLaUsneTgzC7rAgAAeKl6GpgckOT7SVZPMiHJZUkO7K1JAQDAEkspV0s93ZXr8SQf6OW5AAAA/VS3gUkp5UvdvFxrrUcv4vkAAAD90IIyJq3OLFk+yX5JVkoiMAEAAF6ybgOTWuuxLzwvpQxNckiSfZOcleTY+d0HAADMR7XGpJUFrjEppayY5NOZtcbkjCSb1Fqf7O2JAQAA/ceC1ph8O8muScYk2bDWOrUtswIAAPqVBWVM/jfJs0m+kOTzpZQX2ktmLX4f1otzAwCAJU7tqE1PoU9a0BqTHp8MDwAA8GIJPAAAgMb19OR3AABgUXDye0syJgAAQOMEJgAAQOOUcgEAQDs5YLElGRMAAKBxAhMAAKBxSrkAAKCdHLDYkowJAADQOIEJAADQOKVcAADQTg5YbEnGBAAAaJzABAAAaJzABAAAaJw1JgAA0E7WmLQkYwIAACyUUsr2pZRxpZTxpZTDWrz+2VLKzbMffy+lzCylrNjdmAITAACgx0opA5OckGSHJBsk2auUskHXPrXWb9daN6q1bpTk8CRX1Fqf6G5cpVwAANBOdbE/+X2zJONrrfclSSnlrCQ7J7ljPv33SvKLBQ0qYwIAAHQqpYwupYzt8hg9V5fVkzzU5XrC7LZWYy2XZPskv17Q+8qYAAAAnWqtY5KM6aZLaXXbfPq+J8k1CyrjSgQmAADQXov/rlwTkqzZ5XqNJBPn0/f96UEZV6KUCwAAWDjXJxlVShlZSlk6s4KPC+fuVEoZnuRtSS7oyaAyJgAAQI/VWmeUUg5KcmmSgUlOr7XeXko5YPbrJ8/uukuSy2qt03oyrsAEAADaqWOx35UrtdaLk1w8V9vJc13/OMmPezqmUi4AAKBxAhMAAKBxSrkAAKCd6mK/K1evkDEBAAAaJzABAAAaJzABAAAaZ40JAAC00xKwXXBvkDEBAAAaJzABAAAap5QLAADaqHbYLrgVGRMAAKBxAhMAAKBxSrkAAKCd7MrVkowJAADQOIEJAADQOKVcAADQTtWuXK3ImAAAAI0TmAAAAI1TygUAAO1kV66WZEwAAIDGCUwAAIDGKeUCAIB26rArVysyJgAAQOMEJgAAQOMEJgAAQOOsMQEAgHayXXBLMiYAAEDjBCYAAEDjlHIBAEA7VdsFtyJjAgAANE5gAgAANE4pFwAAtJNduVqSMQEAABonMAEAABqnlAsAANqodtiVqxUZEwAAoHECEwAAoHFKuQAAoJ3sytWSjAkAANA4gQkAANA4gQkAANA4a0wAAKCdrDFpScYEAABonMAEAABonFIuAABop+rk91ZkTAAAgMYJTAAAgMYp5QIAgHayK1dLMiYAAEDjBCYAAEDjlHIBAEAbVaVcLcmYAAAAjROYAAAAjVPKBQAA7aSUqyUZEwAAoHECEwAAoHECEwAAoHHWmAAAQDt1dDQ9gz5JxgQAAGicwAQAAGicUi4AAGgn2wW3JGMCAAA0TmACAAA0TikXAAC0k1KulmRMAACAxglMAACAxinlAgCANqpVKVcrMiYAAEDjBCYAAEDjlHIBAEA72ZWrJRkTAACgcQITAACgcUq5AACgnZRytSRjAgAANE5gAgAANE5gAgAANK7X15jsvMlBvf0WQAsTL/hc01OAfmnoDl9uegpAH1etMWlJxgQAAGicwAQAAGic7YIBAKCdlHK1JGMCAAA0TmACAAA0TikXAAC0U0fTE+ibZEwAAICFUkrZvpQyrpQyvpRy2Hz6vL2UcnMp5fZSyhULGlPGBAAA6LFSysAkJyTZJsmEJNeXUi6std7Rpc+IJCcm2b7W+mApZZUFjSswAQCANloCDljcLMn4Wut9SVJKOSvJzknu6NJn7yTn1lofTJJa678WNKhSLgAAoFMpZXQpZWyXx+i5uqye5KEu1xNmt3W1bpIVSil/LqXcUErZZ0HvK2MCAAB0qrWOSTKmmy6l1W1zXQ9K8vokWydZNsm1pZS/1lrvnt+gAhMAAGinxb+Ua0KSNbtcr5FkYos+j9dapyWZVkq5Msnrksw3MFHKBQAALIzrk4wqpYwspSyd5P1JLpyrzwVJ3lpKGVRKWS7JG5Pc2d2gMiYAAECP1VpnlFIOSnJpkoFJTq+13l5KOWD26yfXWu8spVyS5NbMOrnl1Frr37sbV2ACAAAslFrrxUkunqvt5Lmuv53k2z0dU2ACAADt5OT3lqwxAQAAGicwAQAAGqeUCwAA2mgJOPm9V8iYAAAAjROYAAAAjVPKBQAA7WRXrpZkTAAAgMYJTAAAgMYp5QIAgDayK1drMiYAAEDjBCYAAEDjlHIBAEA72ZWrJRkTAACgcQITAACgcUq5AACgjapSrpZkTAAAgMYJTAAAgMYJTAAAgMZZYwIAAO1kjUlLMiYAAEDjBCYAAEDjlHIBAEAb2S64NRkTAACgcQITAACgcUq5AACgnZRytSRjAgAANE5gAgAANE4pFwAAtJFduVqTMQEAABonMAEAABqnlAsAANpIKVdrMiYAAEDjBCYAAEDjBCYAAEDjrDEBAIA2ssakNRkTAACgcQITAACgcUq5AACgnWppegZ9kowJAADQOIEJAADQOKVcAADQRnblak3GBAAAaJzABAAAaJxSLgAAaKPaYVeuVmRMAACAxglMAACAxinlAgCANrIrV2syJgAAQOMEJgAAQOMEJgAAQOOsMQEAgDaq1XbBrciYAAAAjROYAAAAjVPKBQAAbWS74NZkTAAAgMYJTAAAgMYp5QIAgDaqHXblakXGBAAAaJzABAAAaJxSLgAAaKNam55B3yRjAgAANE5gAgAANE4pFwAAtJFduVqTMQEAABonMAEAABqnlAsAANpIKVdrMiYAAEDjBCYAAEDjBCYAAEDjrDEBAIA2cvJ7azImAABA4wQmAABA45RyAQBAG9kuuDUZEwAAoHECEwAAoHFKuQAAoI1qVcrViowJAADQOIEJAADQOKVcAADQRrWj6Rn0TTImAABA4wQmAABA4wQmAADQRh219OlHT5RSti+ljCuljC+lHNbi9beXUiaXUm6e/fjSgsa0xgQAAOixUsrAJCck2SbJhCTXl1IurLXeMVfXq2qt7+7puDImAADAwtgsyfha63211ueSnJVk55c6qMAEAADoVEoZXUoZ2+Uxeq4uqyd5qMv1hNltc3tzKeWWUsrvSimvWdD7KuUCAIA26usnv9daxyQZ002XVh+gznV9Y5JX1lqnllJ2THJ+klHdva+MCQAAsDAmJFmzy/UaSSZ27VBrnVJrnTr7+cVJliqlrNzdoAITAABgYVyfZFQpZWQpZfKAfLUAACAASURBVOkk709yYdcOpZTVSill9vPNMivu+Hd3gyrlAgCANqodfbuUa0FqrTNKKQcluTTJwCSn11pvL6UcMPv1k5PsnuRjpZQZSZ5O8v5a69zlXnMQmAAAAAtldnnWxXO1ndzl+fFJjl+YMZVyAQAAjZMxAQCANuq+oKn/kjEBAAAaJzABAAAap5QLAADaaHHflau3yJgAAACNE5gAAACNU8oFAABt1FGVcrUiYwIAADROYAIAADROKRcAALRRVcrVkowJAADQOIEJAADQOIEJAADQOGtMAACgjWptegZ9k4wJAADQOIEJAADQOKVcAADQRk5+b03GBAAAaJzABAAAaJxSLgAAaCMnv7cmYwIAADROYNIPrDlqzXz9F1/PuePOzU+u/0k++OkPZsCA7v+nH7TUoHzkiI/kW+d8K+fdfV4ufvDi+fbd86A98+Nrf5zz7z4/P7joB9lky00W9UeAJca9D/87o79/dt50yPezzeEn58TfXJOZHR0LvO/2Bx7JAT84J2/77AnZ8jMn5KPfPzu33f9wG2YMfd/664/KZZf8MlMmjc+D/7ghRx35mQX+O5ckw4YNzak/PC6PPXp7/v3YnTnzjP/LiiuuMEefd2791vz0Jydk/N1/zYzn/pkvffHT8x3vve/dIdf+5aI8NXl8Hn3477noNz/Ncsst+5I/H/QXApMl3JDhQ/L1n389tdYcvf/R+cX3f5FdR++aD376g93et8yyy2S7vbbLs08/mztvuHO+/fY4cI/sfcje+e2Zv81X9v9KHrznwRx5+pEZ9dpRi/qjwGJvyvRncsAPzk5J8t0Dds7oHd6cM/84Nif99i/d3vfIE1Py0R+ck5kdHTl6nx3y1Q/tkBkdHfnY/52Tif+e0p7JQx81YsTwXPq7s1Jrza677Zuvfu17+dQnP5qjjvzMAu/9xc9Oytu2fHNGH/DZfGT/T2XTTTfKueecNkef7bZ9RzbccP386fKrM23a9PmO9ZF998pPzzw+l156ed79nv+X0Qd8JveMvz+DBqmaZ1619u1HU3xblnA7fnDHLD146Xx19Ffz9NSnc9NVN2W5Ictl70/tnbNPPjtPT3265X3TpkzLnhvumSR594fenY0232iePoOWGpQ9Pr5Hzj7p7Jxz0jlJkhuvvDFrjlozH/jUB3LUvkf12ueCxdHZV96SZ56bkWNH75Qhyy6TrJ9MfebZnHLRtfnwNm+Y1dbCVX+/P9OfeS7Hjt4pw5YbnCTZaO2X5+2fPTFX335f9thy3u8n9BcfHf3/suyyg7P7HvvnqaemJn+8KsOGDcmXvvi/+fZ3TpzV1sKb3vj6bLfdO/KOrXbNVVf/LUky8Z+P5Nq/XJStt3pr/vinq5Iknzvs6Hz20K8kSXZ6z3Ytx1pppRVy7HeOyiGf/GJOO/3nne0XXHDJovyosMSTMVnCbfr2TXPjFTfOEYBcceEVGbzs4Gz4pg1f0tgve+XLstzQ5XLz1TfP0X7TVTdl4y02zqClxL3Q1TV33J+3bLDWHAHI9pu+Os88PyM33DNhvvfNmDkzAwcOyHLLLN3ZtuwyS2XgwAGN/mYL+oLtt3tHLvv9FXMEIL/81QVZbrll87Yt3zz/+7Z/Rx555F+dQUmSXD/25tx33wPZfrt3dLbVHnzJ3rf7TkmSM39y9ov5CMBsApMl3Bprr5EJ9875A89jEx/LM9OfyZprr/mSxl5qmaWSJM8///wc7c8/93yWWmaprPaK1V7S+LCkuf+RJ7LWqivO0fayFYdl8NKDcv+jT8z3vq03XjeDlxqUY3/95zzx1PQ88dT0fOecP2fYcstkm03W7e1pQ5+23nrrZNy48XO0PfTQxEybNj3rrbf2Qt2XJHfdNT7rrbfOQs1hs802zri7781H9t0r/7hvbJ6e9o/85erf5M1v2nShxqH/6KilTz+aIjBZwg0ZPiRTp8ybxp46eWqGDB/yksZ+5MFH0tHRkXVfN+cPRuu9br0kydARQ1/S+LCkeWr6sxm63LzlWsOWG5wp05+Z732rjBiSH35yj/zx5nuy1aEnZatDT8qfbr4nJx60W1YculxvThn6vBVWGJ5Jk+Zda/Xkk5Ozwgoj5n/fiOGZNLnFfZMmZYUVhi/UHFZb9b+y3rpr54jDD87hn/9a3rvLhzNt2tO56Lc/zSqrrLxQY0F/1uPApJTyylLKO2c/X7aU4qfOxUWrLHTpWXq6O9Ofmp4rLrgiex60Z1775tdmyPAhec+H35ONtphV794xc8E7DUF/UzLvb6Jqbd3+gscmT81nT/1NNnjFqjnhwF1zwoG7Zv01V80nTjwvDz9h8Tu0+ves9ODfudb3lYX+93HAgAEZOnRIRn/0M/nFL87LpZf9Obvu/pHMnDkzB35834UaC/qzHgUmpZT/SXJOklNmN62R5Pxu+o8upYwtpYx9cOqDL32WvGhTJ0/N8sOWn6d9+aHLZ9qUaS95/FO+fEoeuuehHPPLY/Kr236V3T66W876v7OSJE8+/uRLHh+WJEOXWyZPPT1vZmTqM60zKS844/djM2NmR779P+/J5q8Zmc1fMzLHjt4pAweUnPmHsb05ZejznnxyckaMGDZP+/DhwzJp0uT53zdpckYMnzczMmJ46wxMd554clKS5M9XXNvZ9tRTU3Pjjbdl/fXtUgk91dPVyQcm2SzJ35Kk1npPKWWV+XWutY5JMiZJdnzFjpZmNmjCvROyxtprzNG28stWzrLLL5uH7n3oJY8/5YkpOXyvw7PSaitl+WHLZ8K9E/Le/d6bJ/71RP414V8veXxYkoxcbcX8Y661JI88MSVPP/t8Rs619qSr+x99Imu/bKUsNXBgZ9tSgwbmVS9bOQ89NqnX5guLg3Hj5l0TssYaL8+QIctn3Lh7u71vi803m6d9vfXWzoUXXrpQc7jrrnvS0dGRUubMfJZS0tHhxyDm5eT31npayvVsrfW5Fy5KKYPSukCIPmbsn8fm9W97fZZd/j8HPG35ni3zzNPP5La/3rbI3uffj/w7D979YAYOGpht99w2l/3yskU2NiwpNt9gZP5yxwOZ9kznf05z6Q3jMnipQXn9qDXme9/LVxyWex9+PM/PmNnZ9tzzM3LvxMfz8pXm/U0x9CeXXHp5tt3mbRky5D/VAXu87z2ZPv3pXHHltfO/75LL87KXrZrN3/KGzrbXb/LarL32Wrnk0ssXag4XXfSHDBgwIO94+1s624YNG5pNNtkwt956x0KNBf1ZTwOTK0opRyRZtpSyTZKzk/ym96bFonLxTy/O8889ny+M+UI22mKjbL/39vnApz6Q8394/hxbCJ965ak55FuHzHHvpm/fNJvvuHnW3mDWriab77h5Nt9x86yy+n+SZVvtulW22WObbPimDbPVblvluPOOS8fMjvzqhF+15wPCYuR9W74uSw8amE+PuSB/veuBnHP1rTn54mvzwa1fP8cWwu858rQc9ZP//MZ2l803zGOTpuVTp1yQK2+7L1fedm8+dcoFeXzytOy2xWub+CjQZ5wy5id59tnncs6vTs3WW701++/3gXzpi/+b731/zBxbCN91x9UZc8p3Oq//+rcbcumll+dHp38/733vDtlpp+1y5pnH5+qr/9Z5hkmSvOIVq2fXXd+VXXd9V5Zeeqmsv/662XXXd82xpfANN96aCy68JGNO+U7+3/97X3bcYeucf+6P8vzzM3LiST9uy98DLAl6Wsp1WJL9ktyW5KNJLk5yam9NikVn6uSpOXyvw/Pxr3w8R55+ZKZNmZbzTz0/P/vuz+boN3DgwAwYOGeceuDXDsyqa67aef35kz+fJDnu08flD+f8IUlSBpS872Pvyyqrr5JpT03LtZddmzO+eUae6WaHIeivhi03OKcc8r4c88s/5pCTzs/QZZfJB7d6fQ5415xnLcyY2ZGZXRbfbvCKVXPCQbvmlIuvzRfO+F2SZNTLV85JB++e9daYb1Ut9AuTJk3OttvvmR9872s5/7wfZdKkKfn+D36YL3/l2Dn6DRo0KAO7lEMmyd4f/HiO/c5ROXXMsRkwYEAuuvgP+eSnvjhHn7e/bfOcftp3O6/ft/t78r7d35N//OOhrLPumzrb9/nQJ/LNY76Y73zryCy33OD85S9js812e3S7zoX+q8ktefuy0pOdJ0opuyS5uNb67MK+gTUm0Ixf/+i9TU8B+qWhO3y56SlAvzXjuX8uFj/x/+3lu/bpn4/fOPHcRv4ee1rKtVOSu0spPymlvGv2GhMAAIBFokeBSa113yTrZNbakr2T3FtKUcoFAAALqfbxR1N6nPmotT5fSvldZs132SQ7J9m/tyYGAAD0Hz09YHH7UsqPk4xPsntmLXx/WS/OCwAA6Ed6mjH5cJKzknz0xSyABwAAZrErV2s9Ckxqre/v7YkAAAD9V7eBSSnl6lrrFqWUpzLnWpiSpNZaHTkMAAC8ZN0GJrXWLWb/ObQ90wEAgCVbVcrVUk8Xv/+kJ20AAAAvRk8PWHxN14vZByy+ftFPBwAA6I+6DUxKKYfPXl/y2lLKlNmPp5I8muSCtswQAABY4i1ojck3knyjlPKNWuvhbZoTAAAssTqankAf1dPtgg8vpayQZFSSwV3ar+ytiQEAAP1HjwKTUsr+SQ5JskaSm5O8Kcm1SbbqvakBAAD9RU8Xvx+S5A1JHqi1viPJxkke67VZAQDAEqqm9OlHU3oamDxTa30mSUopy9Ra70qyXu9NCwAA6E96VMqVZEIpZUSS85P8vpTyZJKJvTctAACgP+np4vddZj89qpRyeZLhSS7ptVkBAMASqqM2PYO+qaeL31fscnnb7D/9lQIAAItET9eY3JhZi93vTnLP7Of3l1JuLKU4AR4AAHhJerrG5JIk59VaL02SUsq2SbZP8qskJyZ5Y+9MDwAAliwdDe581Zf1NGOy6QtBSZLUWi9LsmWt9a9JlumVmQEAAP1GTzMmT5RSDk1y1uzrPZM8WUoZmKSjV2YGAAD0Gz0NTPZOcmRmbRecJFfPbhuYZI9emBcAACyRmjzEsC/r6XbBjyf5RCllSK116lwvj1/00wIAAPqTHq0xKaW8pZRyR5I7Zl+/rpRyYq/ODAAA6Dd6Wsr13STbJbkwSWqtt5RStuy1WQEAwBLKAu3WerorV2qtD83VNHMRzwUAAOinepoxeaiU8pYktZSydJKDk9zZe9MCAAD6k55mTA5IcmCS1ZNMSLLR7GsAAICXbGF25fpAL88FAACWeLYLbq3bwKSU8qVuXq611qMX8XwAAIB+aEEZk2kt2pZPsl+SlZIITAAAgJes28Ck1nrsC89LKUOTHJJk3yRnJTl2fvcBAACt2S64tQWuMSmlrJjk05m1xuSMJJvUWp/s7YkBAAD9x4LWmHw7ya5JxiTZsNY6tS2zAgAA+pUFZUz+N8mzSb6Q5POldO4gUDJr8fuwXpwbAAAscZRytbagNSY9PhkeAADgxRJ4AAAAjevRAYsAAMCi4YDF1mRMAACAxglMAACAxinlAgCANupQydWSjAkAANA4gQkAANA4gQkAANA4a0wAAKCNOmwX3JKMCQAAsFBKKduXUsaVUsaXUg7rpt8bSikzSym7L2hMgQkAANBjpZSBSU5IskOSDZLsVUrZYD79vpnk0p6MKzABAIA2qn380QObJRlfa72v1vpckrOS7Nyi3yeS/DrJv3oyqMAEAABYGKsneajL9YTZbZ1KKasn2SXJyT0dVGACAAB0KqWMLqWM7fIYPXeXFrfNnWz5XpJDa60ze/q+duUCAIA26mh6AgtQax2TZEw3XSYkWbPL9RpJJs7VZ9MkZ5VSkmTlJDuWUmbUWs+f36ACEwAAYGFcn2RUKWVkkn8meX+Svbt2qLWOfOF5KeXHSX7bXVCSCEwAAICFUGudUUo5KLN22xqY5PRa6+2llANmv97jdSVdCUwAAKCNOsrif8BirfXiJBfP1dYyIKm1frgnY1r8DgAANE5gAgAANE4pFwAAtFEPDzHsd2RMAACAxglMAACAxinlAgCANurrByw2RcYEAABonMAEAABonMAEAABonDUmAADQRh2L/8HvvULGBAAAaJzABAAAaJxSLgAAaKOOqOVqRcYEAABonMAEAABonFIuAABoo9r0BPooGRMAAKBxAhMAAKBxSrkAAKCNHLDYmowJAADQOIEJAADQOKVcAADQRh1NT6CPkjEBAAAaJzABAAAaJzABAAAaZ40JAAC0kZPfW5MxAQAAGicwAQAAGqeUCwAA2sjJ763JmAAAAI0TmAAAAI1TygUAAG3k5PfWZEwAAIDGCUwAAIDGKeUCAIA2UsrVmowJAADQOIEJAADQOKVcAADQRtUBiy3JmAAAAI0TmAAAAI0TmAAAAI2zxgQAANrIdsGtyZgAAACNE5gAAACNU8oFAABtpJSrNRkTAACgcQITAACgcUq5AACgjWrTE+ijZEwAAIDGCUwAAIDGKeUCAIA26ihNz6BvkjEBAAAaJzABAAAap5QLAADayAGLrcmYAAAAjROYAAAAjVPKBQAAbaSUqzUZEwAAoHECEwAAoHECEwAAoHHWmAAAQBvVpifQR8mYAAAAjROYAAAAjVPKBQAAbdRRmp5B3yRjAgAANE5gAgAANE4pFwAAtJGT31uTMQEAABonMAEAABqnlAsAANrIAYutyZgAAACNE5gAAACNU8oFAABt1KGYqyUZEwAAoHG9njF5+LnJvf0WQAuv2PXYpqcA/dJTf/ha01MAWCzJmAAAAI2zxgQAANrIye+tyZgAAACNE5gAAACNU8oFAABtZLPg1mRMAACAxglMAACAxinlAgCANrIrV2syJgAAwEIppWxfShlXShlfSjmsxes7l1JuLaXcXEoZW0rZYkFjypgAAAA9VkoZmOSEJNskmZDk+lLKhbXWO7p0+2OSC2uttZTy2iS/SvLq7sYVmAAAQBt1lKZn8JJtlmR8rfW+JCmlnJVk5ySdgUmtdWqX/sunB5uRKeUCAAA6lVJGzy6/euExeq4uqyd5qMv1hNltc4+zSynlriQXJfnIgt5XxgQAAOhUax2TZEw3XVrlfObJiNRaz0tyXillyyRHJ3lnd+8rMAEAgDbqWPyPWJyQZM0u12skmTi/zrXWK0spa5dSVq61Pj6/fkq5AACAhXF9klGllJGllKWTvD/JhV07lFLWKaWU2c83SbJ0kn93N6iMCQAA0GO11hmllIOSXJpkYJLTa623l1IOmP36yUl2S7JPKeX5JE8n2bPW2m2qSGACAABttNgXciWptV6c5OK52k7u8vybSb65MGMq5QIAABonMAEAABonMAEAABpnjQkAALRRR9MT6KNkTAAAgMYJTAAAgMYp5QIAgDZaAk5+7xUyJgAA/P/27jzKrqrMG/BvJwRCRloQERAICEFmBBRkVNAGPhzACRvtVlAGG1EmG9EWJwQaURRBBttGWmUSUBEEZJJ5RkBEGRKQ0WbMyJCk9vdH3cRKcpNUIHVPknqetWpV3XPP3nefrHVS9Z733XtD4wQmAABA45RyAQBABynkak/GBAAAaJzABAAAaJxSLgAA6CAbLLYnYwIAADROYAIAADROKRcAAHSQDRbbkzEBAAAaJzABAAAaJzABAAAaZ44JAAB0kBkm7cmYAAAAjROYAAAAjVPKBQAAHWTn9/ZkTAAAgMYJTAAAgMYp5QIAgA6q1uVqS8YEAABonMAEAABonFIuAADoIKtytSdjAgAANE5gAgAANE4pFwAAdFCXVbnakjEBAAAaJzABAAAaJzABAAAaZ44JAAB0kBkm7cmYAAAAjROYAAAAjVPKBQAAHWS54PZkTAAAgMYJTAAAgMYp5QIAgA7qanoACykZEwAAoHECEwAAoHFKuQAAoIOqVbnakjEBAAAaJzABAAAap5QLAAA6yKpc7cmYAAAAjROYAAAAjVPKBQAAHWRVrvZkTAAAgMYJTAAAgMYJTAAAgMaZYwIAAB1kueD2ZEwAAIDGCUwAAIDGKeUCAIAO6qqWC25HxgQAAGicwAQAAGicUi4AAOgghVztyZgAAACNE5gAAACNU8oFAAAd1KWYqy0ZEwAAoHECEwAAoHFKuQAAoIOqUq62ZEwAAIDGCUwAAIDGCUwAAIDGmWMCAAAd1NX0ABZSMiYAAEDjBCYAAEDjlHIBAEAH2fm9PRkTAACgcQITAACgcUq5AACgg+z83p6MCQAA0DiBCQAA0DilXAAA0EE2WGxPxgQAAGicwAQAAGicwAQAADqo1rpQf/VGKWXHUspfSykPllIOa/P+HqWUu1tfN5RSNpxXnwITAACg10opA5OcmGSnJOsk+VgpZZ1ZThubZNta6wZJvpnk1Hn1KzABAADmx9uSPFhrHVNrfSXJWUne3/OEWusNtdbnWy9vSrLyvDq1KhcAAHRQ10K+wWIpZe8ke/c4dGqttWfGY6Ukj/Z4/ViSt8+ly72S/G5enyswAQAAZmgFIXMrvSrtmrU9sZR3pjsw2WpenyswAQAA5sdjSd7U4/XKSZ6Y9aRSygZJfpxkp1rrs/Pq1BwTAABgftyaZM1SyqhSypJJdk/ym54nlFJWSXJ+kk/UWu/vTacyJgAA0EGL+s7vtdappZT9k1yaZGCSn9Ra7y2l7Nt6/+QkX02ybJKTSilJMrXWuunc+hWYAAAA86XWenGSi2c5dnKPnz+d5NPz06dSLgAAoHEyJgAA0EF1IV8uuCkyJgAAQOMEJgAAQOOUcgEAQAct7Du/N0XGBAAAaJzABAAAaJxSLgAA6KBalXK1I2MCAAA0TmACAAA0TikXAAB0UFfTA1hIyZgAAACNE5gAAACNE5gAAACNM8cEAAA6qNr5vS0ZEwAAoHECEwAAoHFKuQAAoIO6lHK1JWMCAAA0TmACAAA0TilXP7D6Wqvli0cemA02WS8Tx0/MBb+4MKd85yfp6przvqNLDFoi+39pn6z/1nWzzoZrZ/DSS2XjFbac7bx9D90r79p527xx5RVSSskjD/0tPz3pF7ns11f05SXBQmmt0WvkqGP/M5tutlHGj5uQn51xbo49+odzvdeSZPiIYTnyqMOz0y47ZEAZkMsuvTqHf/Fbef75F2Y675/+aZl8+YgDs9PO22f4iOF57NEncvxxJ+ecs36dJNnoretnz70+ls3fsWnesMLyeeLxp3LeuRfmhONPy8svv9Jn1w0Lo4eeeCbHnHlZ7h7zeIYvPTi7br1h9nnvVhk4YO7PZO99+MmccMHVue+Rp1Jr8pZVV8j+H9gm66++0oxzbvzz2Pz6urty15gn8uSz47LPe7fKfu/buq8vicVIrUq52hGYLOaGjxyek8/5fsbcPzYHfvKwvGm1lXLQ1/ZPKSUnHXPaHNsNXnpwdv2XXfKnO+/LXbfdk7dvvWnb84YOG5oLz744Y+5/ONOmdWWHXbbLMad8I13TpuXy317dR1cFC5+Ry4zIeb8+PX/964P513/5bFYbtUq+/q3/yIABA3LUt46fa9sf/8/xWWPNUTnwc19JV1dXvvr1Q3LGL07Me3faY8Y5w4YPzW9+97NMmjQ5X/rit/Lss89n9Og1suSSg2ac84Hddspqo1bJD44/LWMeeiTrrjs6h33581l3vdH51CcO6LNrh4XN+EkvZt/vnpnVV1wu3/v3D+Wx/3s+x517Zbq6avbfdds5tnvqufHZ57tn5i2rrJBv7vneJMlPL705+x1/ds45Yq+suOzIJMkNfxqT+x97Om9fe9Vccut9Hbkm6A8EJou5D//rB7LU4CVz8J6HZ9LEybn5mlszdPiQ7HPwXvnpiT/PpImT27abOH5itl17pyTJR/f84BwDk+OO+MFMr2/6wy1ZY/So7PLhnQQm9Cuf3HP3DF56qXzy4/tn4oRJ+cNVN2T48GE59LD9c8L3T8vECZPattt0s43yrh22zvt22iM33nBbkuTJJ/6ey676ZbbZbotcc/WNSZIDD943Sy21ZN693Qfz0ksvJ0muv/bmmfo64Xun5dlnn5/x+obrbslLL7+c737/m1n5TSvmsUef6ItLh4XOuX+4My9NmZrj9tstw5ZeKllnVCa+9EpOufDafHLHzbuPtXHt3Q9m8kuv5LjP7pYRQwYnSTZaY6Vsd+D3c909D+Uj2701SXLgh96Vgz+yfZLk6rse6MxFQT9gjslibst3bZ4br75lpgDk0l9dkaWHDM4mW2zcJ5857vnxWWKQmJf+ZfsdtslVV1w3UwBywXkXZciQpfOOLd8253bv3ib/9/enZwQlSXLnHffk4YcfzfY7bDPj2O577Jaf/+8vZwQl7fQMSqa75+7up7nLvX7Z+boeWJRd/6cxece6o2YKQHbc7C156ZWpuf3+v82x3dRpXRk4cECGLLXkjGNLL7VkBg4cMFPpzYABpW8GTr/RlbpQfzVFYLKYW23NVTP2wUdmOvbU43/Pi5NfzGprrrrAPmfgwIEZNmJYdtrtPdl8283yyzN+tcD6hkXBm9daPQ/cP2amY48/9mQmTZqcNddafY7t1mzTLkke+OtDM9qtsurKWX755TJu3IScee6pefzpe3LfQzfmG0celkGDBs3Wtqe3vW3jTJs2LQ8+MPtnwOJq7FPPZrUVZg7G37jsyAxeclDGPvXsHNttv8noDF5yUI4794o8N35Snhs/Kd855/KMGDI47970LX09bOj3PNZezA0fOTwTxk2c7fj4FyZkxMjhC+Qz1n/rujnj4lOTJFOmTM0xh383V19y7QLpGxYVyywzIuPHTZjt+LgXxmeZZUbMsd3IZUZkXJt2L7wwPquutnKSZPnll0uSHPH1Q3PB+Rflox/8TNZdb3S+/NWDMnXatHzjq8e27Xv55ZfLFw7ZN+ee9es5lpLB4mjC5JcyfMjs5VojhgzO+EkvzbHd8ssMz2kH/0sOOOHcnHlFdxbz9SOH5aQvfDSvGz6kz8YLdOtVYFJKWSvJj5K8EoOzFAAAEqlJREFUoda6XillgyTvq7V+q09HxwLRbuWHUsoCWxHigb88lD3+ea8MHzEsW+3wjvzHtw/KpAmTcsmvLl8g/cOi4tXea3NqNz2bPqC1itBf/vJADjrgP5Mk111zU4YNG5YvHLxPjj3qhLz44sx/bA0aNCg/Pv34TJo0OV85/KhXczmwSCuZvdyqpnbfW3Pw9AsTc+jJ52edVVfIEf/WPc/y7KvuyOd+cG5+etgn8sbW5Hd4raoNFtvqbSnXaUm+lGRKktRa706y+5xOLqXsXUq5rZRy2zOTn3rto+RVmzBuQoaPHDbb8WEjhmbC+NkzKa/GS5Nfyp/v+ktuvva2HHfED3LRLy/JAV/Zb4H0DYuKF14Y3zYLOWLEsLYZkenGvTA+I0fOnlEZOXJ4xo0bnyQzlg2edbL7ddfclMGDl8pqo1aZrf2JpxyT0W95cz72ob0z7oXx83UtsKgbPmRwJkyePTMy8cWX22ZSpvvppTdlalfNsfvumi3XWyNbrrdGjttvtwwcUHLGZTfPsR2wYPQ2MBlSa71llmNT53RyrfXUWuumtdZNlxuywqsfHa/Zww88klFvnnkuyRtWXD5Dhg7Jww88ModWr81f7rk/b1x5hSyxxMA+6R8WRg/eP2a2uSQrrrRChg4b2nYOyXQP3D8ma641arbjPeesPDz20bb7kEx/8DvrPinfOurw7Ljz9vnXj/27uSX0S6NWWDYPzzKX5KnnxufFl6dk1ApzXghi7FPPZo0Vl8ugHr+/Bi0xMKuvuFweffqFObYDFozeBibPlFLWSKuwoJTyoSRP9tmoWGCuv/KmbLHd2zNk6D9qY9/z/u3z4uSXcvuNd/bJZ2602fp56vG/Z+rUaX3SPyyMrrj8mrxz+60ydNjQGcc+sNvOmTz5xdxw/azPdXq0+/01ecMKy+ftm28y49iGG6+XUaNWyRWXX5MkmTJlSv5w1fXZapvNZ2q79bZbZNKkyRk75h8PGT5/0N759D4fz2f3PjQ333T7gro8WKRsud7queHesZnUYxW7S2+9L4OXXCKbrDV7hnG6FZcdmYcefzpTevz+emXK1Dz0xDMz9jAB+k5vJ7//e5JTk6xdSnk8ydgke8y9CQuDc8/4VXb/9Idy3E++ndN/+LOstOqK2feQPfOzU86aaQnhX994du648c58/aCjZxzb8l2bZ+khgzN63TWTJDvssl2S5N4/3pcnH/t73rjyG/K147+cS87/fR575PEMGTok79x5m+y467tz5BfbT8aFxdXpPzkrn9nnEzn9ZyfkhONPy6qrvSlfPGz/nHzi6TNNPL/lzstyw/W35gv7fzlJctutf8yVl1+bH55yTL72lWNaGywemptuuG3GHiZJ8p1jTsxvL/1FfnDit3P+eRdlnXVH54AD9853jz0pr7wyJUmy24d2yVeOODhn/uy8PPnE37PJphvOaP/w2L+1XU4YFkcf3nbjnHnlbTnopPPzqR03z2PPvJCTL7w2H9/hbTMtIfzew3+UTdZaJV/75P9Lkuy69Ya54Lq7cuBJ53XvWVJrzr76jjwzbmI+uM1GM9o98ey43Ptw9/PZKVOnZcwTz+T3t/8lSy85KFutv0ZnL5ZFUped39sqvZkAXUoZWGudVkoZmmRArXXOBdOz2HiFLf3LN2z1tVbLf3z7oGywyXqZMH5CfvXz3+bk7/z3TOUfF936y9x2w5054vNHznRsxTe9cbb+vvr5I3Ph2Rdn2PChOeyog7Px2zfIsq9/XSaMn5gx9z+c//3Rmbnuihtna0dnPfbiM00Pod9Za/QaOfo7X82mm22U8ePG52dn/DL/ddQJM91rt999RW647pZ87rNfmnFsxMjh+da3v5Sdd3l3BgwYkMsuvSqHf/HIPPfczIHEO7ffKl854qCMXnvNPPP0sznj9HPyve/8aMbk+RNOOiq777Fb27F9br/DctYvLuiDq2ZWf7vwS/M+iT730BPP5OhfXJa7xzye4UOWyq5bbZh937d1Bg74R7HIToedlE3XWiXf3HOXGcduvu/hnHLhdXnw8aeTJGuu/Prs+76ts9nof5RF//r6u3PE6RfN9plvXHZkfnf0Z/vwqpiXpbf55CKxycw2K22/UP99fM3jVzTy79jbwORvSS5JcnaSK+t8LOckMIFmCEygGQITaI7AZMFoKjDp7RyT0UkuT3dJ19hSyg9LKVv13bAAAGDxVBfyr6b0KjCptb5Yaz2n1rpbko2TjEjyhz4dGQAA0G/0NmOSUsq2pZSTktyRZHCSj/TZqAAAgH6ltzu/j03yxyTnJDm01jppHk0AAIA2uuz83lZvlwvesNZq62AAAKBPzDUwKaV8sdb6X0mOLKXMFtrVWg/os5EBAAD9xrwyJve1vt/W1wMBAID+QClXe3MNTGqtF7Z+nFxrPbfne6WUD/fZqAAAgH6lt6tytdstyg5SAADAAjGvOSY7Jdk5yUqllB/0eGtEkql9OTAAAFgc1aqUq515zTF5It3zS96X5PYexyckObCvBgUAAPQv85pjcleSu0opP6+1ypAAAAB9Yl6lXOfUWj+S5M5ZlgsuSWqtdYM+HR0AACxmrMrV3rxKuT7f+r5LXw8EAADov+a6Klet9cnWj88kebTW+kiSpZJsmO75JwAAAK9Zb5cLvibJ4FLKSkmuSPKpJKf31aAAAID+ZV6lXNOVWuvkUspeSU6otf5XKeXOvhwYAAAsjqo5Jm31NmNSSilbJNkjyUWtY70NagAAAOaqt4HJF9K90/sFtdZ7SymrJ7mq74YFAAD0J73KetRa/5DkD6WU4aWUYbXWMUkO6NuhAQDA4sfO7+31KmNSSlm/NafkT0n+XEq5vZSybt8ODQAA6C96W8p1SpKDaq2r1lpXSXJwktP6blgAAEB/0tsJ7ENrrTPmlNRary6lDO2jMQEAwGLLzu/t9TYwGVNK+c8k/9t6/fEkY/tmSAAAQH/T21KuPZO8Psn5ra/l0r3JIgAAwGs214xJKWVwkn2TvDnJPUkOrrVO6cTAAABgcWRVrvbmlTH5aZJN0x2U7JTk2D4fEQAA0O/Ma47JOrXW9ZOklPLfSW7p+yEBAAD9zbwCkxllW7XWqaWUPh4OAAAs3qzK1d68ApMNSynjWz+XJEu3XpcktdY6ok9HBwAA9AtzDUxqrQM7NRAAAKD/6u1ywQAAAH2mtxssAgAAC0A1x6QtGRMAAKBxAhMAAKBxSrkAAKCDuuz83paMCQAA0DiBCQAA0DilXAAA0EFW5WpPxgQAAGicwAQAAGicUi4AAOggq3K1J2MCAAA0TmACAAA0TikXAAB0kFW52pMxAQAAGicwAQAAGqeUCwAAOsiqXO3JmAAAAI0TmAAAAI0TmAAAAPOllLJjKeWvpZQHSymHtXl/7VLKjaWUl0sph/SmT3NMAACggxb15YJLKQOTnJjk3UkeS3JrKeU3tdY/9zjtuSQHJPlAb/uVMQEAAObH25I8WGsdU2t9JclZSd7f84Ra6//VWm9NMqW3nQpMAACAGUope5dSbuvxtfcsp6yU5NEerx9rHXtNlHIBAEAHLezLBddaT01y6lxOKe2avdbPlTEBAADmx2NJ3tTj9cpJnnitnQpMAACA+XFrkjVLKaNKKUsm2T3Jb15rp0q5AACggxb1VblqrVNLKfsnuTTJwCQ/qbXeW0rZt/X+yaWUFZLclmREkq5SyheSrFNrHT+nfgUmAADAfKm1Xpzk4lmOndzj56fSXeLVa0q5AACAxsmYAABAB9Xa1fQQFkoyJgAAQOMEJgAAQOOUcgEAQAd1LeKrcvUVGRMAAKBxAhMAAKBxAhMAAKBx5pgAAEAH1WqOSTsyJgAAQOMEJgAAQOOUcgEAQAdZLrg9GRMAAKBxAhMAAKBxSrkAAKCDrMrVnowJAADQOIEJAADQOKVcAADQQV1KudqSMQEAABonMAEAABqnlAsAADqo2mCxLRkTAACgcQITAACgcQITAACgceaYAABAB9n5vT0ZEwAAoHECEwAAoHFKuQAAoIO6LBfclowJAADQOIEJAADQOKVcAADQQVblak/GBAAAaJzABAAAaJxSLgAA6KAupVxtyZgAAACNE5gAAACNU8oFAAAdZFWu9mRMAACAxglMAACAxinlAgCADuqKUq52ZEwAAIDGCUwAAIDGCUwAAIDGmWMCAAAdZLng9mRMAACAxglMAACAxinlAgCADupSytWWjAkAANA4gQkAANA4pVwAANBB1c7vbcmYAAAAjROYAAAAjVPKBQAAHWRVrvZkTAAAgMYJTAAAgMYp5QIAgA6qSrnakjEBAAAaJzABAAAaJzABAAAaZ44JAAB0kJ3f25MxAQAAGicwAQAAGqeUCwAAOshywe3JmAAAAI0TmAAAAI1TygUAAB2klKs9GRMAAKBxAhMAAKBxSrkAAKCDFHK1J2MCAAA0TmACAAA0rlgVgLkppexdaz216XFAf+Peg2a496A5MibMy95NDwD6KfceNMO9Bw0RmAAAAI0TmAAAAI0TmDAv6myhGe49aIZ7Dxpi8jsAANA4GRMAAKBxAhMAAKBxApPFVCmlllKO6/H6kFLK115lX8uUUj77Kts+XEpZ7tW0hUXFgrzf5vE5h8/y+oYF/RmwqCqlTCul/LGU8qdSyrmllCHz2X7FUsovWz9vVErZucd77yulHLagxwzMTGCy+Ho5yW4LKChYJknbwKSUMnAB9A+LugV5v83NTIFJrfUdffx5sCh5sda6Ua11vSSvJNl3fhrXWp+otX6o9XKjJDv3eO83tdajF9xQgXYEJouvqeleWeTAWd8opby+lHJeKeXW1teWreNfK6Uc0uO8P5VSVktydJI1Wk+iji2lbFdKuaqU8osk97TO/VUp5fZSyr2lFJtT0d+8mvvt9aWU35dS7iilnFJKeWR6YNPufiqlHJ1k6dZ9+PPWsYmt72fP8nT39FLKB0spA1v37K2llLtLKfv0+b8ELByuTfLmUsrrWvfT3aWUm0opGyRJKWXb1r30x1LKnaWU4aWU1Vq/95ZM8o0kH229/9FSyidLKT8spYxsVQIMaPUzpJTyaCllUClljVLKJa1799pSytoNXj8skgQmi7cTk+xRShk5y/HvJ/lerXWzJB9M8uN59HNYkodaT6IObR17W5Iv11rXab3es9a6SZJNkxxQSll2wVwCLDLm9347IsmVtda3JrkgySo92sx2P9VaD8s/ngjvMctnnJXko0nS+qNq+yQXJ9krybjWZ2+W5DOllFEL6HphoVRKWSLJTul+cPb1JHfWWjdId8bxjNZphyT591rrRkm2TvLi9Pa11leSfDXJ2a377ewe741LcleSbVuH3pvk0lrrlHQ/nPhc6949JMlJfXeVsHhaoukB0HdqreNLKWckOSA9/tNNskOSdUop01+PKKUMn8/ub6m1ju3x+oBSyq6tn9+UZM0kz76KYcMi6VXcb1sl2bXV9pJSyvM92szv/fS7JD8opSyVZMck19RaXyylvCfJBqWU6eUpI1t9jZ1DP7AoW7qU8sfWz9cm+e8kN6f7gUBqrVeWUpZtPTy4Psl3W9nH82utj/W4R+fl7HQ/CLgqye5JTiqlDEvyjiTn9uhnqQVwTdCvCEwWf8cnuSPJ//Q4NiDJFrXWnn88pZQyNTNn0QbPpd9JPdptl+4/vraotU4upVw9j7awuJqf+63tX0Gv5n6qtb7UOu+f0/0H05nTu0v3E9xL5/tKYNHzYisDMsMc7rNaaz26lHJRuueR3FRK2SHJS738nN8kOaqU8rokmyS5MsnQJC/M+vnA/FHKtZirtT6X5Jx0l3RMd1mS/ae/KKVM/4/04SRvbR17a5LpJR8TkswtozIyyfOtP6LWTrL5Ahk8LGLm8367LslHWsfek+SfWsfndj9NKaUMmsPHn5XkU+kuS5keiFyaZL/pbUopa5VShr7Ky4NF0TVJ9khmBP3PtLKba9Ra76m1HpPktiSzzgeZ4++9WuvEJLeku0zzt7XWabXW8UnGllI+3PqsUkrZsE+uCBZjApP+4bgkPVcLOiDJpq3JgH/OP1YuOS/J61qp8P2S3J8ktdZnk1zfmhR4bJv+L0myRCnl7iTfTHJTH10HLAp6e799Pcl7Sil3pLse/sl0/zE0t/vp1CR3T5/8PovLkmyT5PJWjXzSPZ/lz0nuKKX8KckpkSmnf/laWvdfuhdy+bfW8S+0fqfdle7Sy9/N0u6qdJdg/rGU8tE2/Z6d5OOt79PtkWSvVp/3Jnn/grsM6B9KrbXpMQD0O635INNqrVNLKVsk+ZEyEAD6M0/OAJqxSpJzWsuOvpLkMw2PBwAaJWMCAAA0zhwTAACgcQITAACgcQITAACgcQITAACgcQITAACgcf8f25oEynJ9IyAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x1080 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "conf_matrix = pd.DataFrame(matrix, index = ['Neutral','Negative','Positive'],columns = ['Neutral','Negative','Positive'])\n",
    "#Normalizing\n",
    "conf_matrix = conf_matrix.astype('float') / conf_matrix.sum(axis=1)[:, np.newaxis]\n",
    "plt.figure(figsize = (15,15))\n",
    "sns.heatmap(conf_matrix, annot=True, annot_kws={\"size\": 15})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 12.906403,
     "end_time": "2020-10-06T22:00:59.275554",
     "exception": false,
     "start_time": "2020-10-06T22:00:46.369151",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Again, the model's score is average, but keep in mind it hasn't gone through hyperparameter tuning. Let's see how it performs on some test text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:01:26.949303Z",
     "iopub.status.busy": "2020-10-06T22:01:26.947906Z",
     "iopub.status.idle": "2020-10-06T22:01:26.951809Z",
     "shell.execute_reply": "2020-10-06T22:01:26.951229Z"
    },
    "papermill": {
     "duration": 14.303017,
     "end_time": "2020-10-06T22:01:26.951936",
     "exception": false,
     "start_time": "2020-10-06T22:01:12.648919",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentiment = ['Neutral','Negative','Positive']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:01:53.611240Z",
     "iopub.status.busy": "2020-10-06T22:01:53.610198Z",
     "iopub.status.idle": "2020-10-06T22:01:53.680730Z",
     "shell.execute_reply": "2020-10-06T22:01:53.679943Z"
    },
    "papermill": {
     "duration": 13.274625,
     "end_time": "2020-10-06T22:01:53.680880",
     "exception": false,
     "start_time": "2020-10-06T22:01:40.406255",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Negative'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence = tokenizer.texts_to_sequences(['this experience has been the worst , want my money back'])\n",
    "test = pad_sequences(sequence, maxlen=max_len)\n",
    "sentiment[np.around(best_model.predict(test), decimals=0).argmax(axis=1)[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:02:21.215521Z",
     "iopub.status.busy": "2020-10-06T22:02:21.214548Z",
     "iopub.status.idle": "2020-10-06T22:02:21.272639Z",
     "shell.execute_reply": "2020-10-06T22:02:21.271230Z"
    },
    "papermill": {
     "duration": 14.445195,
     "end_time": "2020-10-06T22:02:21.272790",
     "exception": false,
     "start_time": "2020-10-06T22:02:06.827595",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Positive'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence = tokenizer.texts_to_sequences(['this data science article is the best ever'])\n",
    "test = pad_sequences(sequence, maxlen=max_len)\n",
    "sentiment[np.around(best_model.predict(test), decimals=0).argmax(axis=1)[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:02:48.026609Z",
     "iopub.status.busy": "2020-10-06T22:02:48.025563Z",
     "iopub.status.idle": "2020-10-06T22:02:48.086453Z",
     "shell.execute_reply": "2020-10-06T22:02:48.085808Z"
    },
    "papermill": {
     "duration": 13.396502,
     "end_time": "2020-10-06T22:02:48.086621",
     "exception": false,
     "start_time": "2020-10-06T22:02:34.690119",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Negative'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence = tokenizer.texts_to_sequences(['Shivam is a bad boy'])\n",
    "test = pad_sequences(sequence, maxlen=max_len)\n",
    "sentiment[np.around(best_model.predict(test), decimals=0).argmax(axis=1)[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-06T22:03:14.961947Z",
     "iopub.status.busy": "2020-10-06T22:03:14.960920Z",
     "iopub.status.idle": "2020-10-06T22:03:15.014373Z",
     "shell.execute_reply": "2020-10-06T22:03:15.013738Z"
    },
    "papermill": {
     "duration": 13.646468,
     "end_time": "2020-10-06T22:03:15.014510",
     "exception": false,
     "start_time": "2020-10-06T22:03:01.368042",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Positive'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence = tokenizer.texts_to_sequences(['i really loved how the technician helped me with the issue that i had'])\n",
    "test = pad_sequences(sequence, maxlen=max_len)\n",
    "sentiment[np.around(best_model.predict(test), decimals=0).argmax(axis=1)[0]]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "duration": 2724.650172,
   "end_time": "2020-10-06T22:04:52.151005",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-10-06T21:19:27.500833",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}