NamePredict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data_path = 'data/data.csv'\n",
    "all_data = pd.read_csv(data_path, sep=',')\n",
    "pd.set_option('max_rows', 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_dict = {1:'Company',\n",
    "              2:'EducationalInstitution',\n",
    "              3:'Artist',\n",
    "              4:'Athlete',\n",
    "              5:'OfficeHolder',\n",
    "              6:'MeanOfTransportation',\n",
    "              7:'Building',\n",
    "              8:'NaturalPlace',\n",
    "              9:'Village',\n",
    "              10:'Animal',\n",
    "              11:'Plant',\n",
    "              12:'Album',\n",
    "              13:'Film',\n",
    "              14:'WrittenWork'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Name</th>\n",
       "      <th>Class_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>E. D. Abbott Ltd</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Schwan-Stabilo</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Q-workshop</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Marvell Software Solutions Israel</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Bergan Mercy Medical Center</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>The Unsigned Guide</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>Rest of the world</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>Globoforce</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>Rompetrol</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>Wave Accounting</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>Angstrem (company)</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>I-innovate (UK)</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>JVC</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>Toei Bus</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>Tear Drop Records</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548772</th>\n",
       "      <td>14</td>\n",
       "      <td>HaOlam HaZeh</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548773</th>\n",
       "      <td>14</td>\n",
       "      <td>The OMAC Project</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548774</th>\n",
       "      <td>14</td>\n",
       "      <td>Fledgling (novel)</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548775</th>\n",
       "      <td>14</td>\n",
       "      <td>Death of a Salesman</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548776</th>\n",
       "      <td>14</td>\n",
       "      <td>The Wild Geese (Carney novel)</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548777</th>\n",
       "      <td>14</td>\n",
       "      <td>Fluide Glacial</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548778</th>\n",
       "      <td>14</td>\n",
       "      <td>Poison in Jest</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548779</th>\n",
       "      <td>14</td>\n",
       "      <td>Out of This Furnace</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548780</th>\n",
       "      <td>14</td>\n",
       "      <td>Bush's Brain</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548781</th>\n",
       "      <td>14</td>\n",
       "      <td>Risk (magazine)</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548782</th>\n",
       "      <td>14</td>\n",
       "      <td>The Worst Band in the Universe</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548783</th>\n",
       "      <td>14</td>\n",
       "      <td>Poor Arnold's Almanac</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548784</th>\n",
       "      <td>14</td>\n",
       "      <td>The Girls' Guide to Hunting and Fishing</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548785</th>\n",
       "      <td>14</td>\n",
       "      <td>Wash This Blood Clean from My Hand</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548786</th>\n",
       "      <td>14</td>\n",
       "      <td>Virginia Environmental Law Journal</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>548787 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Class                                     Name   Class_name\n",
       "0           1                         E. D. Abbott Ltd      Company\n",
       "1           1                           Schwan-Stabilo      Company\n",
       "2           1                               Q-workshop      Company\n",
       "3           1        Marvell Software Solutions Israel      Company\n",
       "4           1              Bergan Mercy Medical Center      Company\n",
       "5           1                       The Unsigned Guide      Company\n",
       "6           1                        Rest of the world      Company\n",
       "7           1                               Globoforce      Company\n",
       "8           1                                Rompetrol      Company\n",
       "9           1                          Wave Accounting      Company\n",
       "10          1                       Angstrem (company)      Company\n",
       "11          1                          I-innovate (UK)      Company\n",
       "12          1                                      JVC      Company\n",
       "13          1                                 Toei Bus      Company\n",
       "14          1                        Tear Drop Records      Company\n",
       "...       ...                                      ...          ...\n",
       "548772     14                             HaOlam HaZeh  WrittenWork\n",
       "548773     14                         The OMAC Project  WrittenWork\n",
       "548774     14                        Fledgling (novel)  WrittenWork\n",
       "548775     14                      Death of a Salesman  WrittenWork\n",
       "548776     14            The Wild Geese (Carney novel)  WrittenWork\n",
       "548777     14                           Fluide Glacial  WrittenWork\n",
       "548778     14                           Poison in Jest  WrittenWork\n",
       "548779     14                      Out of This Furnace  WrittenWork\n",
       "548780     14                             Bush's Brain  WrittenWork\n",
       "548781     14                          Risk (magazine)  WrittenWork\n",
       "548782     14           The Worst Band in the Universe  WrittenWork\n",
       "548783     14                    Poor Arnold's Almanac  WrittenWork\n",
       "548784     14  The Girls' Guide to Hunting and Fishing  WrittenWork\n",
       "548785     14       Wash This Blood Clean from My Hand  WrittenWork\n",
       "548786     14       Virginia Environmental Law Journal  WrittenWork\n",
       "\n",
       "[548787 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data['Class_name'] = all_data['Class'].map(class_dict)\n",
    "all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove Annotations (anything between parenthesis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Name</th>\n",
       "      <th>Class_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>E. D. Abbott Ltd</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Schwan-Stabilo</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Q-workshop</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Marvell Software Solutions Israel</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Bergan Mercy Medical Center</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>The Unsigned Guide</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>Rest of the world</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>Globoforce</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>Rompetrol</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>Wave Accounting</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>Angstrem</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>I-innovate</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>JVC</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>Toei Bus</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>Tear Drop Records</td>\n",
       "      <td>Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548772</th>\n",
       "      <td>14</td>\n",
       "      <td>HaOlam HaZeh</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548773</th>\n",
       "      <td>14</td>\n",
       "      <td>The OMAC Project</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548774</th>\n",
       "      <td>14</td>\n",
       "      <td>Fledgling</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548775</th>\n",
       "      <td>14</td>\n",
       "      <td>Death of a Salesman</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548776</th>\n",
       "      <td>14</td>\n",
       "      <td>The Wild Geese</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548777</th>\n",
       "      <td>14</td>\n",
       "      <td>Fluide Glacial</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548778</th>\n",
       "      <td>14</td>\n",
       "      <td>Poison in Jest</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548779</th>\n",
       "      <td>14</td>\n",
       "      <td>Out of This Furnace</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548780</th>\n",
       "      <td>14</td>\n",
       "      <td>Bush's Brain</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548781</th>\n",
       "      <td>14</td>\n",
       "      <td>Risk</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548782</th>\n",
       "      <td>14</td>\n",
       "      <td>The Worst Band in the Universe</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548783</th>\n",
       "      <td>14</td>\n",
       "      <td>Poor Arnold's Almanac</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548784</th>\n",
       "      <td>14</td>\n",
       "      <td>The Girls' Guide to Hunting and Fishing</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548785</th>\n",
       "      <td>14</td>\n",
       "      <td>Wash This Blood Clean from My Hand</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548786</th>\n",
       "      <td>14</td>\n",
       "      <td>Virginia Environmental Law Journal</td>\n",
       "      <td>WrittenWork</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>548787 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Class                                     Name   Class_name\n",
       "0           1                         E. D. Abbott Ltd      Company\n",
       "1           1                           Schwan-Stabilo      Company\n",
       "2           1                               Q-workshop      Company\n",
       "3           1        Marvell Software Solutions Israel      Company\n",
       "4           1              Bergan Mercy Medical Center      Company\n",
       "5           1                       The Unsigned Guide      Company\n",
       "6           1                        Rest of the world      Company\n",
       "7           1                               Globoforce      Company\n",
       "8           1                                Rompetrol      Company\n",
       "9           1                          Wave Accounting      Company\n",
       "10          1                                Angstrem       Company\n",
       "11          1                              I-innovate       Company\n",
       "12          1                                      JVC      Company\n",
       "13          1                                 Toei Bus      Company\n",
       "14          1                        Tear Drop Records      Company\n",
       "...       ...                                      ...          ...\n",
       "548772     14                             HaOlam HaZeh  WrittenWork\n",
       "548773     14                         The OMAC Project  WrittenWork\n",
       "548774     14                               Fledgling   WrittenWork\n",
       "548775     14                      Death of a Salesman  WrittenWork\n",
       "548776     14                          The Wild Geese   WrittenWork\n",
       "548777     14                           Fluide Glacial  WrittenWork\n",
       "548778     14                           Poison in Jest  WrittenWork\n",
       "548779     14                      Out of This Furnace  WrittenWork\n",
       "548780     14                             Bush's Brain  WrittenWork\n",
       "548781     14                                    Risk   WrittenWork\n",
       "548782     14           The Worst Band in the Universe  WrittenWork\n",
       "548783     14                    Poor Arnold's Almanac  WrittenWork\n",
       "548784     14  The Girls' Guide to Hunting and Fishing  WrittenWork\n",
       "548785     14       Wash This Blood Clean from My Hand  WrittenWork\n",
       "548786     14       Virginia Environmental Law Journal  WrittenWork\n",
       "\n",
       "[548787 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "all_data['Name'] = [re.sub(r'\\([^)]*\\)', '', x) for x in all_data['Name']]\n",
    "all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load USE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import numpy as np\n",
    "import tf_sentencepiece\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "module_url = 'module_Multi_Large/'\n",
    "\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    similarity_input_placeholder = tf.placeholder(tf.string, shape=(None))\n",
    "    embed = hub.Module(module_url)\n",
    "    encoding_tensor = embed(similarity_input_placeholder)\n",
    "    init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])\n",
    "g.finalize()\n",
    "\n",
    "sess = tf.Session(graph=g)\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def embed_df(df, col):\n",
    "    message_embedding = []\n",
    "    size_batch = 200\n",
    "    data_size = len(df[col])\n",
    "    num_batches = int(data_size/size_batch)+1\n",
    "    for x in tqdm(range(num_batches)):\n",
    "        samples = list(df[col][(x*size_batch):((x+1)*size_batch)])\n",
    "        if type(samples) is pd.core.series.Series:\n",
    "            message_embedding.extend(sess.run(encoding_tensor, feed_dict={similarity_input_placeholder: [samples]}))\n",
    "        else:\n",
    "            message_embedding.extend(sess.run(encoding_tensor, feed_dict={similarity_input_placeholder: samples}))\n",
    "\n",
    "    df[col+'_emb'] = message_embedding\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "load = True\n",
    "file_path = 'complete_data.pkl'\n",
    "\n",
    "if load == True:\n",
    "    with open(file_path, 'rb') as fl:\n",
    "        all_data = pickle.load(fl)\n",
    "else:\n",
    "    columns = ['Name']\n",
    "    for col in columns:\n",
    "        all_data = embed_df(all_data, col)\n",
    "    \n",
    "    with open(file_path, 'wb') as fs:\n",
    "        pickle.dump(all_data , fs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check Data Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     40000\n",
       "2     40000\n",
       "3     40000\n",
       "4     40000\n",
       "5     40000\n",
       "6     40000\n",
       "7     40000\n",
       "8     40000\n",
       "9     40000\n",
       "10    40000\n",
       "11    40000\n",
       "12    40000\n",
       "13    40000\n",
       "14    28787\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data['Class'].value_counts().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split into Train, Validate and Test (Stratified)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "\n",
    "class my_dataset(Dataset):\n",
    "    def __init__(self, data):\n",
    "        input_ = [torch.tensor(x).float() for x in data['Name_emb'].values]\n",
    "        label_ = [torch.tensor(x).long() for x in data['Class'].values]\n",
    "        self.samples = list(zip(input_, label_))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.samples)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.samples[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Length: \n",
      " -train (90%):\t\t 439029 \n",
      " -validate (10%):\t 54879 \n",
      " -test (10%):\t\t 54879\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = all_data.drop('Class', axis=1)\n",
    "y = all_data['Class']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2)\n",
    "X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, stratify=y_test, test_size=0.5)\n",
    "\n",
    "train = pd.concat([X_train, y_train.reindex(X_train.index)], axis=1)\n",
    "validate = pd.concat([X_val, y_val.reindex(X_val.index)], axis=1)\n",
    "test = pd.concat([X_test, y_test.reindex(X_test.index)], axis=1)\n",
    "\n",
    "trainset = my_dataset(train)\n",
    "validateset = my_dataset(validate)\n",
    "testset = my_dataset(test)\n",
    "print('Dataset Length: \\n -train (90%%):\\t\\t %d \\n -validate (10%%):\\t %d \\n -test (10%%):\\t\\t %d' % (len(trainset), len(validateset), len(testset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 5000\n",
    "\n",
    "trainloader = data.DataLoader(trainset, batch_size=batch_size, num_workers=0, shuffle=True)\n",
    "validateloader = data.DataLoader(validateset, batch_size=batch_size, num_workers=0, shuffle=True)\n",
    "testloader = data.DataLoader(testset, batch_size=batch_size, num_workers=0, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (fc1): Linear(in_features=512, out_features=500, bias=True)\n",
       "  (fc2): Sequential(\n",
       "    (0): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1): Linear(in_features=500, out_features=500, bias=True)\n",
       "    (2): Dropout(p=0.3, inplace=True)\n",
       "    (3): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (4): Linear(in_features=500, out_features=500, bias=True)\n",
       "    (5): Dropout(p=0.3, inplace=True)\n",
       "  )\n",
       "  (fc3): Sequential(\n",
       "    (0): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1): Linear(in_features=500, out_features=15, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, EMBED_DIM, CLASS_DIM):\n",
    "        super(Net, self).__init__()\n",
    "        self.number_neurons = 500\n",
    "        self.fc1 = nn.Linear(EMBED_DIM, self.number_neurons)\n",
    "        \n",
    "        self.fc2 = nn.Sequential(nn.BatchNorm1d(self.number_neurons),\n",
    "                                 nn.Linear(self.number_neurons, self.number_neurons),\n",
    "                                 nn.Dropout(0.3, inplace=True),\n",
    "                                 \n",
    "                                 nn.BatchNorm1d(self.number_neurons),\n",
    "                                 nn.Linear(self.number_neurons, self.number_neurons),\n",
    "                                 nn.Dropout(0.3, inplace=True)\n",
    "                                )\n",
    "        \n",
    "        self.fc3 = nn.Sequential(\n",
    "                                 nn.BatchNorm1d(self.number_neurons),\n",
    "                                 nn.Linear(self.number_neurons, CLASS_DIM)\n",
    "                                )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        x = F.leaky_relu(self.fc2(x))\n",
    "        x = F.softmax(self.fc3(x), dim=1)\n",
    "        return (x)\n",
    "    \n",
    "EMBED_DIM, CLASS_DIM = trainloader.dataset[0][0].shape[0], len(class_dict)+1\n",
    "\n",
    "net = Net(EMBED_DIM, CLASS_DIM)\n",
    "net.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Evaluate Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(net, data_loader):\n",
    "\n",
    "    loss = []\n",
    "    accuracy = []\n",
    "\n",
    "    y_true = []\n",
    "    y_pred = []\n",
    "    \n",
    "    for input_labels in data_loader:\n",
    "        inputs, labels = input_labels[0].to(device), input_labels[1].to(device)\n",
    "\n",
    "        outputs = net(inputs)\n",
    "\n",
    "        loss.append(criterion(outputs, labels).data.cpu().numpy())\n",
    "        outputs = outputs.to('cpu').detach().numpy() \n",
    "        labels = labels.to('cpu').detach().numpy() \n",
    "\n",
    "        predict = [np.argmax(x) for x in outputs]\n",
    "        labels = np.array([int(x) for x in labels])\n",
    "\n",
    "        accuracy.append(list(predict==labels).count(True)/len(predict))\n",
    "\n",
    "        y_true.extend(labels)\n",
    "        y_pred.extend(predict)\n",
    "        \n",
    "\n",
    "    print('Acc: %.2f'% np.mean(accuracy))\n",
    "    print('Loss: %.2f'% np.mean(loss))\n",
    "    return y_true, y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.5, momentum=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 0\n",
      "[1,    29] loss: 2.376\n",
      "[1,    58] loss: 2.235\n",
      "[1,    87] loss: 2.220\n",
      "--------------- Epoch 1\n",
      "[2,    29] loss: 2.209\n",
      "[2,    58] loss: 2.205\n",
      "[2,    87] loss: 2.200\n",
      "--------------- Epoch 2\n",
      "[3,    29] loss: 2.194\n",
      "[3,    58] loss: 2.193\n",
      "[3,    87] loss: 2.186\n",
      "--------------- Epoch 3\n",
      "[4,    29] loss: 2.178\n",
      "[4,    58] loss: 2.178\n",
      "[4,    87] loss: 2.176\n",
      "--------------- Epoch 4\n",
      "[5,    29] loss: 2.171\n",
      "[5,    58] loss: 2.169\n",
      "[5,    87] loss: 2.172\n",
      "--------------- Epoch 5\n",
      "[6,    29] loss: 2.166\n",
      "[6,    58] loss: 2.164\n",
      "[6,    87] loss: 2.164\n",
      "--------------- Epoch 6\n",
      "[7,    29] loss: 2.159\n",
      "[7,    58] loss: 2.161\n",
      "[7,    87] loss: 2.162\n",
      "--------------- Epoch 7\n",
      "[8,    29] loss: 2.156\n",
      "[8,    58] loss: 2.157\n",
      "[8,    87] loss: 2.158\n",
      "--------------- Epoch 8\n",
      "[9,    29] loss: 2.151\n",
      "[9,    58] loss: 2.155\n",
      "[9,    87] loss: 2.157\n",
      "--------------- Epoch 9\n",
      "[10,    29] loss: 2.150\n",
      "[10,    58] loss: 2.152\n",
      "[10,    87] loss: 2.153\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tValidate:\n",
      "Acc: 0.64\n",
      "Loss: 2.18\n",
      "\tTest:\n",
      "Acc: 0.64\n",
      "Loss: 2.18\n",
      "--------------- Epoch 10\n",
      "[11,    29] loss: 2.148\n",
      "[11,    58] loss: 2.148\n",
      "[11,    87] loss: 2.151\n",
      "--------------- Epoch 11\n",
      "[12,    29] loss: 2.145\n",
      "[12,    58] loss: 2.146\n",
      "[12,    87] loss: 2.147\n",
      "--------------- Epoch 12\n",
      "[13,    29] loss: 2.142\n",
      "[13,    58] loss: 2.143\n",
      "[13,    87] loss: 2.144\n",
      "--------------- Epoch 13\n",
      "[14,    29] loss: 2.139\n",
      "[14,    58] loss: 2.142\n",
      "[14,    87] loss: 2.143\n",
      "--------------- Epoch 14\n",
      "[15,    29] loss: 2.139\n",
      "[15,    58] loss: 2.139\n",
      "[15,    87] loss: 2.142\n",
      "--------------- Epoch 15\n",
      "[16,    29] loss: 2.135\n",
      "[16,    58] loss: 2.139\n",
      "[16,    87] loss: 2.137\n",
      "--------------- Epoch 16\n",
      "[17,    29] loss: 2.132\n",
      "[17,    58] loss: 2.137\n",
      "[17,    87] loss: 2.135\n",
      "--------------- Epoch 17\n",
      "[18,    29] loss: 2.129\n",
      "[18,    58] loss: 2.133\n",
      "[18,    87] loss: 2.136\n",
      "--------------- Epoch 18\n",
      "[19,    29] loss: 2.128\n",
      "[19,    58] loss: 2.130\n",
      "[19,    87] loss: 2.132\n",
      "--------------- Epoch 19\n",
      "[20,    29] loss: 2.128\n",
      "[20,    58] loss: 2.128\n",
      "[20,    87] loss: 2.128\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.69\n",
      "Loss: 2.13\n",
      "\tValidate:\n",
      "Acc: 0.65\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.65\n",
      "Loss: 2.16\n",
      "--------------- Epoch 20\n",
      "[21,    29] loss: 2.125\n",
      "[21,    58] loss: 2.127\n",
      "[21,    87] loss: 2.130\n",
      "--------------- Epoch 21\n",
      "[22,    29] loss: 2.125\n",
      "[22,    58] loss: 2.124\n",
      "[22,    87] loss: 2.124\n",
      "--------------- Epoch 22\n",
      "[23,    29] loss: 2.121\n",
      "[23,    58] loss: 2.123\n",
      "[23,    87] loss: 2.125\n",
      "--------------- Epoch 23\n",
      "[24,    29] loss: 2.119\n",
      "[24,    58] loss: 2.120\n",
      "[24,    87] loss: 2.123\n",
      "--------------- Epoch 24\n",
      "[25,    29] loss: 2.116\n",
      "[25,    58] loss: 2.119\n",
      "[25,    87] loss: 2.122\n",
      "--------------- Epoch 25\n",
      "[26,    29] loss: 2.115\n",
      "[26,    58] loss: 2.120\n",
      "[26,    87] loss: 2.121\n",
      "--------------- Epoch 26\n",
      "[27,    29] loss: 2.115\n",
      "[27,    58] loss: 2.117\n",
      "[27,    87] loss: 2.117\n",
      "--------------- Epoch 27\n",
      "[28,    29] loss: 2.115\n",
      "[28,    58] loss: 2.114\n",
      "[28,    87] loss: 2.115\n",
      "--------------- Epoch 28\n",
      "[29,    29] loss: 2.112\n",
      "[29,    58] loss: 2.116\n",
      "[29,    87] loss: 2.116\n",
      "--------------- Epoch 29\n",
      "[30,    29] loss: 2.111\n",
      "[30,    58] loss: 2.111\n",
      "[30,    87] loss: 2.116\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.70\n",
      "Loss: 2.12\n",
      "\tValidate:\n",
      "Acc: 0.65\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.65\n",
      "Loss: 2.16\n",
      "--------------- Epoch 30\n",
      "[31,    29] loss: 2.109\n",
      "[31,    58] loss: 2.113\n",
      "[31,    87] loss: 2.111\n",
      "--------------- Epoch 31\n",
      "[32,    29] loss: 2.106\n",
      "[32,    58] loss: 2.110\n",
      "[32,    87] loss: 2.114\n",
      "--------------- Epoch 32\n",
      "[33,    29] loss: 2.108\n",
      "[33,    58] loss: 2.108\n",
      "[33,    87] loss: 2.112\n",
      "--------------- Epoch 33\n",
      "[34,    29] loss: 2.105\n",
      "[34,    58] loss: 2.110\n",
      "[34,    87] loss: 2.111\n",
      "--------------- Epoch 34\n",
      "[35,    29] loss: 2.106\n",
      "[35,    58] loss: 2.108\n",
      "[35,    87] loss: 2.109\n",
      "--------------- Epoch 35\n",
      "[36,    29] loss: 2.105\n",
      "[36,    58] loss: 2.106\n",
      "[36,    87] loss: 2.109\n",
      "--------------- Epoch 36\n",
      "[37,    29] loss: 2.101\n",
      "[37,    58] loss: 2.106\n",
      "[37,    87] loss: 2.108\n",
      "--------------- Epoch 37\n",
      "[38,    29] loss: 2.102\n",
      "[38,    58] loss: 2.104\n",
      "[38,    87] loss: 2.105\n",
      "--------------- Epoch 38\n",
      "[39,    29] loss: 2.102\n",
      "[39,    58] loss: 2.101\n",
      "[39,    87] loss: 2.103\n",
      "--------------- Epoch 39\n",
      "[40,    29] loss: 2.099\n",
      "[40,    58] loss: 2.101\n",
      "[40,    87] loss: 2.102\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.72\n",
      "Loss: 2.10\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "--------------- Epoch 40\n",
      "[41,    29] loss: 2.097\n",
      "[41,    58] loss: 2.102\n",
      "[41,    87] loss: 2.102\n",
      "--------------- Epoch 41\n",
      "[42,    29] loss: 2.097\n",
      "[42,    58] loss: 2.099\n",
      "[42,    87] loss: 2.100\n",
      "--------------- Epoch 42\n",
      "[43,    29] loss: 2.094\n",
      "[43,    58] loss: 2.099\n",
      "[43,    87] loss: 2.098\n",
      "--------------- Epoch 43\n",
      "[44,    29] loss: 2.093\n",
      "[44,    58] loss: 2.094\n",
      "[44,    87] loss: 2.098\n",
      "--------------- Epoch 44\n",
      "[45,    29] loss: 2.093\n",
      "[45,    58] loss: 2.095\n",
      "[45,    87] loss: 2.096\n",
      "--------------- Epoch 45\n",
      "[46,    29] loss: 2.091\n",
      "[46,    58] loss: 2.093\n",
      "[46,    87] loss: 2.095\n",
      "--------------- Epoch 46\n",
      "[47,    29] loss: 2.089\n",
      "[47,    58] loss: 2.092\n",
      "[47,    87] loss: 2.093\n",
      "--------------- Epoch 47\n",
      "[48,    29] loss: 2.090\n",
      "[48,    58] loss: 2.091\n",
      "[48,    87] loss: 2.093\n",
      "--------------- Epoch 48\n",
      "[49,    29] loss: 2.089\n",
      "[49,    58] loss: 2.089\n",
      "[49,    87] loss: 2.091\n",
      "--------------- Epoch 49\n",
      "[50,    29] loss: 2.089\n",
      "[50,    58] loss: 2.086\n",
      "[50,    87] loss: 2.090\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.73\n",
      "Loss: 2.09\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "--------------- Epoch 50\n",
      "[51,    29] loss: 2.085\n",
      "[51,    58] loss: 2.090\n",
      "[51,    87] loss: 2.090\n",
      "--------------- Epoch 51\n",
      "[52,    29] loss: 2.086\n",
      "[52,    58] loss: 2.089\n",
      "[52,    87] loss: 2.088\n",
      "--------------- Epoch 52\n",
      "[53,    29] loss: 2.085\n",
      "[53,    58] loss: 2.086\n",
      "[53,    87] loss: 2.088\n",
      "--------------- Epoch 53\n",
      "[54,    29] loss: 2.083\n",
      "[54,    58] loss: 2.083\n",
      "[54,    87] loss: 2.089\n",
      "--------------- Epoch 54\n",
      "[55,    29] loss: 2.083\n",
      "[55,    58] loss: 2.087\n",
      "[55,    87] loss: 2.086\n",
      "--------------- Epoch 55\n",
      "[56,    29] loss: 2.082\n",
      "[56,    58] loss: 2.083\n",
      "[56,    87] loss: 2.087\n",
      "--------------- Epoch 56\n",
      "[57,    29] loss: 2.083\n",
      "[57,    58] loss: 2.083\n",
      "[57,    87] loss: 2.085\n",
      "--------------- Epoch 57\n",
      "[58,    29] loss: 2.081\n",
      "[58,    58] loss: 2.081\n",
      "[58,    87] loss: 2.085\n",
      "--------------- Epoch 58\n",
      "[59,    29] loss: 2.079\n",
      "[59,    58] loss: 2.080\n",
      "[59,    87] loss: 2.082\n",
      "--------------- Epoch 59\n",
      "[60,    29] loss: 2.077\n",
      "[60,    58] loss: 2.080\n",
      "[60,    87] loss: 2.081\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.73\n",
      "Loss: 2.09\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "--------------- Epoch 60\n",
      "[61,    29] loss: 2.077\n",
      "[61,    58] loss: 2.080\n",
      "[61,    87] loss: 2.079\n",
      "--------------- Epoch 61\n",
      "[62,    29] loss: 2.076\n",
      "[62,    58] loss: 2.078\n",
      "[62,    87] loss: 2.078\n",
      "--------------- Epoch 62\n",
      "[63,    29] loss: 2.075\n",
      "[63,    58] loss: 2.077\n",
      "[63,    87] loss: 2.078\n",
      "--------------- Epoch 63\n",
      "[64,    29] loss: 2.073\n",
      "[64,    58] loss: 2.077\n",
      "[64,    87] loss: 2.077\n",
      "--------------- Epoch 64\n",
      "[65,    29] loss: 2.073\n",
      "[65,    58] loss: 2.075\n",
      "[65,    87] loss: 2.075\n",
      "--------------- Epoch 65\n",
      "[66,    29] loss: 2.074\n",
      "[66,    58] loss: 2.073\n",
      "[66,    87] loss: 2.075\n",
      "--------------- Epoch 66\n",
      "[67,    29] loss: 2.071\n",
      "[67,    58] loss: 2.074\n",
      "[67,    87] loss: 2.078\n",
      "--------------- Epoch 67\n",
      "[68,    29] loss: 2.070\n",
      "[68,    58] loss: 2.073\n",
      "[68,    87] loss: 2.075\n",
      "--------------- Epoch 68\n",
      "[69,    29] loss: 2.069\n",
      "[69,    58] loss: 2.071\n",
      "[69,    87] loss: 2.075\n",
      "--------------- Epoch 69\n",
      "[70,    29] loss: 2.070\n",
      "[70,    58] loss: 2.071\n",
      "[70,    87] loss: 2.074\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.74\n",
      "Loss: 2.07\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "--------------- Epoch 70\n",
      "[71,    29] loss: 2.071\n",
      "[71,    58] loss: 2.073\n",
      "[71,    87] loss: 2.072\n",
      "--------------- Epoch 71\n",
      "[72,    29] loss: 2.067\n",
      "[72,    58] loss: 2.071\n",
      "[72,    87] loss: 2.071\n",
      "--------------- Epoch 72\n",
      "[73,    29] loss: 2.067\n",
      "[73,    58] loss: 2.068\n",
      "[73,    87] loss: 2.070\n",
      "--------------- Epoch 73\n",
      "[74,    29] loss: 2.064\n",
      "[74,    58] loss: 2.068\n",
      "[74,    87] loss: 2.070\n",
      "--------------- Epoch 74\n",
      "[75,    29] loss: 2.066\n",
      "[75,    58] loss: 2.068\n",
      "[75,    87] loss: 2.069\n",
      "--------------- Epoch 75\n",
      "[76,    29] loss: 2.067\n",
      "[76,    58] loss: 2.067\n",
      "[76,    87] loss: 2.067\n",
      "--------------- Epoch 76\n",
      "[77,    29] loss: 2.064\n",
      "[77,    58] loss: 2.065\n",
      "[77,    87] loss: 2.068\n",
      "--------------- Epoch 77\n",
      "[78,    29] loss: 2.063\n",
      "[78,    58] loss: 2.067\n",
      "[78,    87] loss: 2.065\n",
      "--------------- Epoch 78\n",
      "[79,    29] loss: 2.061\n",
      "[79,    58] loss: 2.066\n",
      "[79,    87] loss: 2.064\n",
      "--------------- Epoch 79\n",
      "[80,    29] loss: 2.062\n",
      "[80,    58] loss: 2.064\n",
      "[80,    87] loss: 2.066\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.76\n",
      "Loss: 2.06\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "--------------- Epoch 80\n",
      "[81,    29] loss: 2.061\n",
      "[81,    58] loss: 2.064\n",
      "[81,    87] loss: 2.066\n",
      "--------------- Epoch 81\n",
      "[82,    29] loss: 2.061\n",
      "[82,    58] loss: 2.061\n",
      "[82,    87] loss: 2.065\n",
      "--------------- Epoch 82\n",
      "[83,    29] loss: 2.060\n",
      "[83,    58] loss: 2.063\n",
      "[83,    87] loss: 2.064\n",
      "--------------- Epoch 83\n",
      "[84,    29] loss: 2.058\n",
      "[84,    58] loss: 2.061\n",
      "[84,    87] loss: 2.064\n",
      "--------------- Epoch 84\n",
      "[85,    29] loss: 2.060\n",
      "[85,    58] loss: 2.060\n",
      "[85,    87] loss: 2.063\n",
      "--------------- Epoch 85\n",
      "[86,    29] loss: 2.058\n",
      "[86,    58] loss: 2.057\n",
      "[86,    87] loss: 2.063\n",
      "--------------- Epoch 86\n",
      "[87,    29] loss: 2.059\n",
      "[87,    58] loss: 2.060\n",
      "[87,    87] loss: 2.063\n",
      "--------------- Epoch 87\n",
      "[88,    29] loss: 2.058\n",
      "[88,    58] loss: 2.061\n",
      "[88,    87] loss: 2.059\n",
      "--------------- Epoch 88\n",
      "[89,    29] loss: 2.057\n",
      "[89,    58] loss: 2.058\n",
      "[89,    87] loss: 2.060\n",
      "--------------- Epoch 89\n",
      "[90,    29] loss: 2.055\n",
      "[90,    58] loss: 2.057\n",
      "[90,    87] loss: 2.059\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.76\n",
      "Loss: 2.06\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "--------------- Epoch 90\n",
      "[91,    29] loss: 2.055\n",
      "[91,    58] loss: 2.058\n",
      "[91,    87] loss: 2.057\n",
      "--------------- Epoch 91\n",
      "[92,    29] loss: 2.055\n",
      "[92,    58] loss: 2.056\n",
      "[92,    87] loss: 2.057\n",
      "--------------- Epoch 92\n",
      "[93,    29] loss: 2.055\n",
      "[93,    58] loss: 2.054\n",
      "[93,    87] loss: 2.058\n",
      "--------------- Epoch 93\n",
      "[94,    29] loss: 2.052\n",
      "[94,    58] loss: 2.056\n",
      "[94,    87] loss: 2.057\n",
      "--------------- Epoch 94\n",
      "[95,    29] loss: 2.052\n",
      "[95,    58] loss: 2.056\n",
      "[95,    87] loss: 2.056\n",
      "--------------- Epoch 95\n",
      "[96,    29] loss: 2.053\n",
      "[96,    58] loss: 2.055\n",
      "[96,    87] loss: 2.056\n",
      "--------------- Epoch 96\n",
      "[97,    29] loss: 2.052\n",
      "[97,    58] loss: 2.053\n",
      "[97,    87] loss: 2.055\n",
      "--------------- Epoch 97\n",
      "[98,    29] loss: 2.053\n",
      "[98,    58] loss: 2.053\n",
      "[98,    87] loss: 2.053\n",
      "--------------- Epoch 98\n",
      "[99,    29] loss: 2.053\n",
      "[99,    58] loss: 2.052\n",
      "[99,    87] loss: 2.054\n",
      "--------------- Epoch 99\n",
      "[100,    29] loss: 2.051\n",
      "[100,    58] loss: 2.052\n",
      "[100,    87] loss: 2.052\n",
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.76\n",
      "Loss: 2.06\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "display_n_loss = 3\n",
    "display_n_loss = int(len(trainset)/(display_n_loss * batch_size))\n",
    "\n",
    "EPOCHS = 100\n",
    "display_n_epoch = 10\n",
    "\n",
    "for epoch in range(EPOCHS):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    net.train()\n",
    "    print('--------------- Epoch', epoch)\n",
    "    for i, input_labels in enumerate(trainloader, 0):\n",
    "        inputs, labels = input_labels[0].to(device), input_labels[1].to(device)\n",
    "    \n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        \n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        if i % display_n_loss == display_n_loss-1:    # print every N mini-batches\n",
    "            print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / display_n_loss))\n",
    "            running_loss = 0.0\n",
    "            \n",
    "    if epoch % display_n_epoch == display_n_epoch-1:\n",
    "        net.eval()\n",
    "\n",
    "        print('-> Evaluate') \n",
    "        print('\\tTrain:')\n",
    "        evaluate(net, trainloader)\n",
    "        print('\\tValidate:')\n",
    "        evaluate(net, validateloader)\n",
    "        print('\\tTest:')\n",
    "        evaluate(net, testloader)\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> Evaluate\n",
      "\tTrain:\n",
      "Acc: 0.76\n",
      "Loss: 2.06\n",
      "\tValidate:\n",
      "Acc: 0.66\n",
      "Loss: 2.16\n",
      "\tTest:\n",
      "Acc: 0.66\n",
      "Loss: 2.15\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('-> Evaluate') \n",
    "net.eval()\n",
    "print('\\tTrain:')\n",
    "evaluate(net, trainloader)\n",
    "print('\\tValidate:')\n",
    "evaluate(net, validateloader)\n",
    "print('\\tTest:')\n",
    "evaluate(net, testloader)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistics on Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 0.66\n",
      "Loss: 2.15\n"
     ]
    }
   ],
   "source": [
    "from pandas_ml import ConfusionMatrix\n",
    "\n",
    "y_true, y_pred = evaluate(net, testloader)\n",
    "cm = ConfusionMatrix(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1_score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Company</th>\n",
       "      <td>0.707</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.711</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EducationalInstitution</th>\n",
       "      <td>0.947</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.95</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Artist</th>\n",
       "      <td>0.403</td>\n",
       "      <td>0.312</td>\n",
       "      <td>0.352</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Athlete</th>\n",
       "      <td>0.438</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.503</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OfficeHolder</th>\n",
       "      <td>0.507</td>\n",
       "      <td>0.455</td>\n",
       "      <td>0.48</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanOfTransportation</th>\n",
       "      <td>0.908</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.896</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Building</th>\n",
       "      <td>0.851</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.842</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NaturalPlace</th>\n",
       "      <td>0.81</td>\n",
       "      <td>0.796</td>\n",
       "      <td>0.803</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Village</th>\n",
       "      <td>0.679</td>\n",
       "      <td>0.835</td>\n",
       "      <td>0.749</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Animal</th>\n",
       "      <td>0.69</td>\n",
       "      <td>0.659</td>\n",
       "      <td>0.674</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Plant</th>\n",
       "      <td>0.683</td>\n",
       "      <td>0.789</td>\n",
       "      <td>0.732</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Album</th>\n",
       "      <td>0.541</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.567</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Film</th>\n",
       "      <td>0.501</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.448</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WrittenWork</th>\n",
       "      <td>0.549</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.446</td>\n",
       "      <td>2879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Avg_Total</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.663</td>\n",
       "      <td>0.658</td>\n",
       "      <td>54879</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       precision recall F1_score support\n",
       "Company                    0.707  0.716    0.711    4000\n",
       "EducationalInstitution     0.947  0.953     0.95    4000\n",
       "Artist                     0.403  0.312    0.352    4000\n",
       "Athlete                    0.438  0.592    0.503    4000\n",
       "OfficeHolder               0.507  0.455     0.48    4000\n",
       "MeanOfTransportation       0.908  0.885    0.896    4000\n",
       "Building                   0.851  0.833    0.842    4000\n",
       "NaturalPlace                0.81  0.796    0.803    4000\n",
       "Village                    0.679  0.835    0.749    4000\n",
       "Animal                      0.69  0.659    0.674    4000\n",
       "Plant                      0.683  0.789    0.732    4000\n",
       "Album                      0.541  0.595    0.567    4000\n",
       "Film                       0.501  0.405    0.448    4000\n",
       "WrittenWork                0.549  0.376    0.446    2879\n",
       "Avg_Total                   0.66  0.663    0.658   54879"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('precision', 3)\n",
    "\n",
    "cm_df_report = cm.classification_report\n",
    "classes_name = list(class_dict.values())\n",
    "classes_name.append('Avg_Total')\n",
    "\n",
    "cm_df_report.index = classes_name\n",
    "cm_df_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hal9002/.local/lib/python3.5/site-packages/pandas_ml/confusion_matrix/stats.py:60: FutureWarning: supplying multiple axes to axis is deprecated and will be removed in a future version.\n",
      "  num = df[df > 1].dropna(axis=[0, 1], thresh=1).applymap(lambda n: choose(n, 2)).sum().sum() - np.float64(nis2 * njs2) / n2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Company</th>\n",
       "      <th>EducationalInstitution</th>\n",
       "      <th>Artist</th>\n",
       "      <th>Athlete</th>\n",
       "      <th>OfficeHolder</th>\n",
       "      <th>MeanOfTransportation</th>\n",
       "      <th>Building</th>\n",
       "      <th>NaturalPlace</th>\n",
       "      <th>Village</th>\n",
       "      <th>Animal</th>\n",
       "      <th>Plant</th>\n",
       "      <th>Album</th>\n",
       "      <th>Film</th>\n",
       "      <th>WrittenWork</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Population</th>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "      <td>54879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P: Condition positive</th>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>2879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N: Condition negative</th>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>50879</td>\n",
       "      <td>52000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Test outcome positive</th>\n",
       "      <td>4046</td>\n",
       "      <td>4022</td>\n",
       "      <td>3098</td>\n",
       "      <td>5413</td>\n",
       "      <td>3590</td>\n",
       "      <td>3901</td>\n",
       "      <td>3912</td>\n",
       "      <td>3933</td>\n",
       "      <td>4918</td>\n",
       "      <td>3820</td>\n",
       "      <td>4620</td>\n",
       "      <td>4398</td>\n",
       "      <td>3236</td>\n",
       "      <td>1972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Test outcome negative</th>\n",
       "      <td>50833</td>\n",
       "      <td>50857</td>\n",
       "      <td>51781</td>\n",
       "      <td>49466</td>\n",
       "      <td>51289</td>\n",
       "      <td>50978</td>\n",
       "      <td>50967</td>\n",
       "      <td>50946</td>\n",
       "      <td>49961</td>\n",
       "      <td>51059</td>\n",
       "      <td>50259</td>\n",
       "      <td>50481</td>\n",
       "      <td>51643</td>\n",
       "      <td>52907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP: True Positive</th>\n",
       "      <td>2862</td>\n",
       "      <td>3810</td>\n",
       "      <td>1248</td>\n",
       "      <td>2369</td>\n",
       "      <td>1821</td>\n",
       "      <td>3541</td>\n",
       "      <td>3331</td>\n",
       "      <td>3186</td>\n",
       "      <td>3338</td>\n",
       "      <td>2636</td>\n",
       "      <td>3157</td>\n",
       "      <td>2381</td>\n",
       "      <td>1620</td>\n",
       "      <td>1082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TN: True Negative</th>\n",
       "      <td>49695</td>\n",
       "      <td>50667</td>\n",
       "      <td>49029</td>\n",
       "      <td>47835</td>\n",
       "      <td>49110</td>\n",
       "      <td>50519</td>\n",
       "      <td>50298</td>\n",
       "      <td>50132</td>\n",
       "      <td>49299</td>\n",
       "      <td>49695</td>\n",
       "      <td>49416</td>\n",
       "      <td>48862</td>\n",
       "      <td>49263</td>\n",
       "      <td>51110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FP: False Positive</th>\n",
       "      <td>1184</td>\n",
       "      <td>212</td>\n",
       "      <td>1850</td>\n",
       "      <td>3044</td>\n",
       "      <td>1769</td>\n",
       "      <td>360</td>\n",
       "      <td>581</td>\n",
       "      <td>747</td>\n",
       "      <td>1580</td>\n",
       "      <td>1184</td>\n",
       "      <td>1463</td>\n",
       "      <td>2017</td>\n",
       "      <td>1616</td>\n",
       "      <td>890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FN: False Negative</th>\n",
       "      <td>1138</td>\n",
       "      <td>190</td>\n",
       "      <td>2752</td>\n",
       "      <td>1631</td>\n",
       "      <td>2179</td>\n",
       "      <td>459</td>\n",
       "      <td>669</td>\n",
       "      <td>814</td>\n",
       "      <td>662</td>\n",
       "      <td>1364</td>\n",
       "      <td>843</td>\n",
       "      <td>1619</td>\n",
       "      <td>2380</td>\n",
       "      <td>1797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPR: (Sensitivity, hit rate, recall)</th>\n",
       "      <td>0.716</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.312</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.455</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.796</td>\n",
       "      <td>0.835</td>\n",
       "      <td>0.659</td>\n",
       "      <td>0.789</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TNR=SPC: (Specificity)</th>\n",
       "      <td>0.977</td>\n",
       "      <td>0.996</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.94</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.993</td>\n",
       "      <td>0.989</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PPV: Pos Pred Value (Precision)</th>\n",
       "      <td>0.707</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.403</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.507</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.679</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.683</td>\n",
       "      <td>0.541</td>\n",
       "      <td>0.501</td>\n",
       "      <td>0.549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NPV: Neg Pred Value</th>\n",
       "      <td>0.978</td>\n",
       "      <td>0.996</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FPR: False-out</th>\n",
       "      <td>0.0233</td>\n",
       "      <td>0.00417</td>\n",
       "      <td>0.0364</td>\n",
       "      <td>0.0598</td>\n",
       "      <td>0.0348</td>\n",
       "      <td>0.00708</td>\n",
       "      <td>0.0114</td>\n",
       "      <td>0.0147</td>\n",
       "      <td>0.0311</td>\n",
       "      <td>0.0233</td>\n",
       "      <td>0.0288</td>\n",
       "      <td>0.0396</td>\n",
       "      <td>0.0318</td>\n",
       "      <td>0.0171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FDR: False Discovery Rate</th>\n",
       "      <td>0.293</td>\n",
       "      <td>0.0527</td>\n",
       "      <td>0.597</td>\n",
       "      <td>0.562</td>\n",
       "      <td>0.493</td>\n",
       "      <td>0.0923</td>\n",
       "      <td>0.149</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.321</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.317</td>\n",
       "      <td>0.459</td>\n",
       "      <td>0.499</td>\n",
       "      <td>0.451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FNR: Miss Rate</th>\n",
       "      <td>0.284</td>\n",
       "      <td>0.0475</td>\n",
       "      <td>0.688</td>\n",
       "      <td>0.408</td>\n",
       "      <td>0.545</td>\n",
       "      <td>0.115</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.203</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.341</td>\n",
       "      <td>0.211</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACC: Accuracy</th>\n",
       "      <td>0.958</td>\n",
       "      <td>0.993</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.934</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 score</th>\n",
       "      <td>0.711</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.352</td>\n",
       "      <td>0.503</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.896</td>\n",
       "      <td>0.842</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.674</td>\n",
       "      <td>0.732</td>\n",
       "      <td>0.567</td>\n",
       "      <td>0.448</td>\n",
       "      <td>0.446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MCC: Matthews correlation coefficient</th>\n",
       "      <td>0.689</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.464</td>\n",
       "      <td>0.442</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.788</td>\n",
       "      <td>0.731</td>\n",
       "      <td>0.649</td>\n",
       "      <td>0.712</td>\n",
       "      <td>0.532</td>\n",
       "      <td>0.412</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Informedness</th>\n",
       "      <td>0.692</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.276</td>\n",
       "      <td>0.532</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.878</td>\n",
       "      <td>0.821</td>\n",
       "      <td>0.782</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.636</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.556</td>\n",
       "      <td>0.373</td>\n",
       "      <td>0.359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Markedness</th>\n",
       "      <td>0.685</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.465</td>\n",
       "      <td>0.899</td>\n",
       "      <td>0.838</td>\n",
       "      <td>0.794</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.663</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.509</td>\n",
       "      <td>0.455</td>\n",
       "      <td>0.515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Prevalence</th>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>0.0525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LR+: Positive likelihood ratio</th>\n",
       "      <td>30.7</td>\n",
       "      <td>229</td>\n",
       "      <td>8.58</td>\n",
       "      <td>9.9</td>\n",
       "      <td>13.1</td>\n",
       "      <td>125</td>\n",
       "      <td>72.9</td>\n",
       "      <td>54.3</td>\n",
       "      <td>26.9</td>\n",
       "      <td>28.3</td>\n",
       "      <td>27.4</td>\n",
       "      <td>15</td>\n",
       "      <td>12.8</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LR-: Negative likelihood ratio</th>\n",
       "      <td>0.291</td>\n",
       "      <td>0.0477</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.434</td>\n",
       "      <td>0.564</td>\n",
       "      <td>0.116</td>\n",
       "      <td>0.169</td>\n",
       "      <td>0.207</td>\n",
       "      <td>0.171</td>\n",
       "      <td>0.349</td>\n",
       "      <td>0.217</td>\n",
       "      <td>0.421</td>\n",
       "      <td>0.615</td>\n",
       "      <td>0.635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DOR: Diagnostic odds ratio</th>\n",
       "      <td>106</td>\n",
       "      <td>4.79e+03</td>\n",
       "      <td>12</td>\n",
       "      <td>22.8</td>\n",
       "      <td>23.2</td>\n",
       "      <td>1.08e+03</td>\n",
       "      <td>431</td>\n",
       "      <td>263</td>\n",
       "      <td>157</td>\n",
       "      <td>81.1</td>\n",
       "      <td>126</td>\n",
       "      <td>35.6</td>\n",
       "      <td>20.7</td>\n",
       "      <td>34.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FOR: False omission rate</th>\n",
       "      <td>0.0224</td>\n",
       "      <td>0.00374</td>\n",
       "      <td>0.0531</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.0425</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.0131</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.0133</td>\n",
       "      <td>0.0267</td>\n",
       "      <td>0.0168</td>\n",
       "      <td>0.0321</td>\n",
       "      <td>0.0461</td>\n",
       "      <td>0.034</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Company EducationalInstitution  Artist  \\\n",
       "Population                              54879                  54879   54879   \n",
       "P: Condition positive                    4000                   4000    4000   \n",
       "N: Condition negative                   50879                  50879   50879   \n",
       "Test outcome positive                    4046                   4022    3098   \n",
       "Test outcome negative                   50833                  50857   51781   \n",
       "TP: True Positive                        2862                   3810    1248   \n",
       "TN: True Negative                       49695                  50667   49029   \n",
       "FP: False Positive                       1184                    212    1850   \n",
       "FN: False Negative                       1138                    190    2752   \n",
       "TPR: (Sensitivity, hit rate, recall)    0.716                  0.953   0.312   \n",
       "TNR=SPC: (Specificity)                  0.977                  0.996   0.964   \n",
       "PPV: Pos Pred Value (Precision)         0.707                  0.947   0.403   \n",
       "NPV: Neg Pred Value                     0.978                  0.996   0.947   \n",
       "FPR: False-out                         0.0233                0.00417  0.0364   \n",
       "FDR: False Discovery Rate               0.293                 0.0527   0.597   \n",
       "FNR: Miss Rate                          0.284                 0.0475   0.688   \n",
       "ACC: Accuracy                           0.958                  0.993   0.916   \n",
       "F1 score                                0.711                   0.95   0.352   \n",
       "MCC: Matthews correlation coefficient   0.689                  0.946    0.31   \n",
       "Informedness                            0.692                  0.948   0.276   \n",
       "Markedness                              0.685                  0.944    0.35   \n",
       "Prevalence                             0.0729                 0.0729  0.0729   \n",
       "LR+: Positive likelihood ratio           30.7                    229    8.58   \n",
       "LR-: Negative likelihood ratio          0.291                 0.0477   0.714   \n",
       "DOR: Diagnostic odds ratio                106               4.79e+03      12   \n",
       "FOR: False omission rate               0.0224                0.00374  0.0531   \n",
       "\n",
       "                                      Athlete OfficeHolder  \\\n",
       "Population                              54879        54879   \n",
       "P: Condition positive                    4000         4000   \n",
       "N: Condition negative                   50879        50879   \n",
       "Test outcome positive                    5413         3590   \n",
       "Test outcome negative                   49466        51289   \n",
       "TP: True Positive                        2369         1821   \n",
       "TN: True Negative                       47835        49110   \n",
       "FP: False Positive                       3044         1769   \n",
       "FN: False Negative                       1631         2179   \n",
       "TPR: (Sensitivity, hit rate, recall)    0.592        0.455   \n",
       "TNR=SPC: (Specificity)                   0.94        0.965   \n",
       "PPV: Pos Pred Value (Precision)         0.438        0.507   \n",
       "NPV: Neg Pred Value                     0.967        0.958   \n",
       "FPR: False-out                         0.0598       0.0348   \n",
       "FDR: False Discovery Rate               0.562        0.493   \n",
       "FNR: Miss Rate                          0.408        0.545   \n",
       "ACC: Accuracy                           0.915        0.928   \n",
       "F1 score                                0.503         0.48   \n",
       "MCC: Matthews correlation coefficient   0.464        0.442   \n",
       "Informedness                            0.532         0.42   \n",
       "Markedness                              0.405        0.465   \n",
       "Prevalence                             0.0729       0.0729   \n",
       "LR+: Positive likelihood ratio            9.9         13.1   \n",
       "LR-: Negative likelihood ratio          0.434        0.564   \n",
       "DOR: Diagnostic odds ratio               22.8         23.2   \n",
       "FOR: False omission rate                0.033       0.0425   \n",
       "\n",
       "                                      MeanOfTransportation Building  \\\n",
       "Population                                           54879    54879   \n",
       "P: Condition positive                                 4000     4000   \n",
       "N: Condition negative                                50879    50879   \n",
       "Test outcome positive                                 3901     3912   \n",
       "Test outcome negative                                50978    50967   \n",
       "TP: True Positive                                     3541     3331   \n",
       "TN: True Negative                                    50519    50298   \n",
       "FP: False Positive                                     360      581   \n",
       "FN: False Negative                                     459      669   \n",
       "TPR: (Sensitivity, hit rate, recall)                 0.885    0.833   \n",
       "TNR=SPC: (Specificity)                               0.993    0.989   \n",
       "PPV: Pos Pred Value (Precision)                      0.908    0.851   \n",
       "NPV: Neg Pred Value                                  0.991    0.987   \n",
       "FPR: False-out                                     0.00708   0.0114   \n",
       "FDR: False Discovery Rate                           0.0923    0.149   \n",
       "FNR: Miss Rate                                       0.115    0.167   \n",
       "ACC: Accuracy                                        0.985    0.977   \n",
       "F1 score                                             0.896    0.842   \n",
       "MCC: Matthews correlation coefficient                0.888     0.83   \n",
       "Informedness                                         0.878    0.821   \n",
       "Markedness                                           0.899    0.838   \n",
       "Prevalence                                          0.0729   0.0729   \n",
       "LR+: Positive likelihood ratio                         125     72.9   \n",
       "LR-: Negative likelihood ratio                       0.116    0.169   \n",
       "DOR: Diagnostic odds ratio                        1.08e+03      431   \n",
       "FOR: False omission rate                             0.009   0.0131   \n",
       "\n",
       "                                      NaturalPlace Village  Animal   Plant  \\\n",
       "Population                                   54879   54879   54879   54879   \n",
       "P: Condition positive                         4000    4000    4000    4000   \n",
       "N: Condition negative                        50879   50879   50879   50879   \n",
       "Test outcome positive                         3933    4918    3820    4620   \n",
       "Test outcome negative                        50946   49961   51059   50259   \n",
       "TP: True Positive                             3186    3338    2636    3157   \n",
       "TN: True Negative                            50132   49299   49695   49416   \n",
       "FP: False Positive                             747    1580    1184    1463   \n",
       "FN: False Negative                             814     662    1364     843   \n",
       "TPR: (Sensitivity, hit rate, recall)         0.796   0.835   0.659   0.789   \n",
       "TNR=SPC: (Specificity)                       0.985   0.969   0.977   0.971   \n",
       "PPV: Pos Pred Value (Precision)               0.81   0.679    0.69   0.683   \n",
       "NPV: Neg Pred Value                          0.984   0.987   0.973   0.983   \n",
       "FPR: False-out                              0.0147  0.0311  0.0233  0.0288   \n",
       "FDR: False Discovery Rate                     0.19   0.321    0.31   0.317   \n",
       "FNR: Miss Rate                               0.203   0.166   0.341   0.211   \n",
       "ACC: Accuracy                                0.972   0.959   0.954   0.958   \n",
       "F1 score                                     0.803   0.749   0.674   0.732   \n",
       "MCC: Matthews correlation coefficient        0.788   0.731   0.649   0.712   \n",
       "Informedness                                 0.782   0.803   0.636    0.76   \n",
       "Markedness                                   0.794   0.665   0.663   0.667   \n",
       "Prevalence                                  0.0729  0.0729  0.0729  0.0729   \n",
       "LR+: Positive likelihood ratio                54.3    26.9    28.3    27.4   \n",
       "LR-: Negative likelihood ratio               0.207   0.171   0.349   0.217   \n",
       "DOR: Diagnostic odds ratio                     263     157    81.1     126   \n",
       "FOR: False omission rate                     0.016  0.0133  0.0267  0.0168   \n",
       "\n",
       "                                        Album    Film WrittenWork  \n",
       "Population                              54879   54879       54879  \n",
       "P: Condition positive                    4000    4000        2879  \n",
       "N: Condition negative                   50879   50879       52000  \n",
       "Test outcome positive                    4398    3236        1972  \n",
       "Test outcome negative                   50481   51643       52907  \n",
       "TP: True Positive                        2381    1620        1082  \n",
       "TN: True Negative                       48862   49263       51110  \n",
       "FP: False Positive                       2017    1616         890  \n",
       "FN: False Negative                       1619    2380        1797  \n",
       "TPR: (Sensitivity, hit rate, recall)    0.595   0.405       0.376  \n",
       "TNR=SPC: (Specificity)                   0.96   0.968       0.983  \n",
       "PPV: Pos Pred Value (Precision)         0.541   0.501       0.549  \n",
       "NPV: Neg Pred Value                     0.968   0.954       0.966  \n",
       "FPR: False-out                         0.0396  0.0318      0.0171  \n",
       "FDR: False Discovery Rate               0.459   0.499       0.451  \n",
       "FNR: Miss Rate                          0.405   0.595       0.624  \n",
       "ACC: Accuracy                           0.934   0.927       0.951  \n",
       "F1 score                                0.567   0.448       0.446  \n",
       "MCC: Matthews correlation coefficient   0.532   0.412        0.43  \n",
       "Informedness                            0.556   0.373       0.359  \n",
       "Markedness                              0.509   0.455       0.515  \n",
       "Prevalence                             0.0729  0.0729      0.0525  \n",
       "LR+: Positive likelihood ratio             15    12.8          22  \n",
       "LR-: Negative likelihood ratio          0.421   0.615       0.635  \n",
       "DOR: Diagnostic odds ratio               35.6    20.7        34.6  \n",
       "FOR: False omission rate               0.0321  0.0461       0.034  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_od = cm.stats()\n",
    "cm_od['class'].columns = class_dict.values()\n",
    "cm_od['class']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_df = cm.to_dataframe()\n",
    "cm_df.columns = class_dict.values()\n",
    "cm_df.index = class_dict.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col0 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col11 {\n",
       "            background-color:  #e3f4f8;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col1 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col2 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col3 {\n",
       "            background-color:  #48b27f;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col4 {\n",
       "            background-color:  #74c8ae;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col2 {\n",
       "            background-color:  #48b27f;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col3 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col4 {\n",
       "            background-color:  #aadfd3;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col2 {\n",
       "            background-color:  #5cbd98;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col3 {\n",
       "            background-color:  #5cbd98;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col4 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col5 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col6 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col7 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col8 {\n",
       "            background-color:  #e3f4f8;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col8 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col9 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col10 {\n",
       "            background-color:  #aadfd3;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col9 {\n",
       "            background-color:  #c5e9e2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col10 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col11 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col12 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col13 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col11 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col12 {\n",
       "            background-color:  #8fd4c2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col13 {\n",
       "            background-color:  #c5e9e2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col8 {\n",
       "            background-color:  #e3f4f8;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col11 {\n",
       "            background-color:  #8fd4c2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col12 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col13 {\n",
       "            background-color:  #8fd4c2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col0 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col1 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col2 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col3 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col4 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col5 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col6 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col7 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col8 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col9 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col10 {\n",
       "            background-color:  #edf8fb;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col11 {\n",
       "            background-color:  #c5e9e2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col12 {\n",
       "            background-color:  #8fd4c2;\n",
       "            color:  #000000;\n",
       "        }    #T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col13 {\n",
       "            background-color:  #005a24;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Company</th>        <th class=\"col_heading level0 col1\" >EducationalInstitution</th>        <th class=\"col_heading level0 col2\" >Artist</th>        <th class=\"col_heading level0 col3\" >Athlete</th>        <th class=\"col_heading level0 col4\" >OfficeHolder</th>        <th class=\"col_heading level0 col5\" >MeanOfTransportation</th>        <th class=\"col_heading level0 col6\" >Building</th>        <th class=\"col_heading level0 col7\" >NaturalPlace</th>        <th class=\"col_heading level0 col8\" >Village</th>        <th class=\"col_heading level0 col9\" >Animal</th>        <th class=\"col_heading level0 col10\" >Plant</th>        <th class=\"col_heading level0 col11\" >Album</th>        <th class=\"col_heading level0 col12\" >Film</th>        <th class=\"col_heading level0 col13\" >WrittenWork</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row0\" class=\"row_heading level0 row0\" >Company</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col0\" class=\"data row0 col0\" >2862</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col1\" class=\"data row0 col1\" >44</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col2\" class=\"data row0 col2\" >58</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col3\" class=\"data row0 col3\" >42</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col4\" class=\"data row0 col4\" >41</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col5\" class=\"data row0 col5\" >93</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col6\" class=\"data row0 col6\" >138</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col7\" class=\"data row0 col7\" >96</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col8\" class=\"data row0 col8\" >137</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col9\" class=\"data row0 col9\" >72</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col10\" class=\"data row0 col10\" >66</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col11\" class=\"data row0 col11\" >214</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col12\" class=\"data row0 col12\" >65</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row0_col13\" class=\"data row0 col13\" >72</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row1\" class=\"row_heading level0 row1\" >EducationalInstitution</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col0\" class=\"data row1 col0\" >43</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col1\" class=\"data row1 col1\" >3810</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col2\" class=\"data row1 col2\" >4</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col3\" class=\"data row1 col3\" >8</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col4\" class=\"data row1 col4\" >20</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col5\" class=\"data row1 col5\" >14</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col6\" class=\"data row1 col6\" >59</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col7\" class=\"data row1 col7\" >8</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col8\" class=\"data row1 col8\" >9</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col9\" class=\"data row1 col9\" >5</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col10\" class=\"data row1 col10\" >2</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col11\" class=\"data row1 col11\" >5</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col12\" class=\"data row1 col12\" >6</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row1_col13\" class=\"data row1 col13\" >7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row2\" class=\"row_heading level0 row2\" >Artist</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col0\" class=\"data row2 col0\" >58</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col1\" class=\"data row2 col1\" >3</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col2\" class=\"data row2 col2\" >1248</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col3\" class=\"data row2 col3\" >1409</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col4\" class=\"data row2 col4\" >787</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col5\" class=\"data row2 col5\" >39</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col6\" class=\"data row2 col6\" >35</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col7\" class=\"data row2 col7\" >61</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col8\" class=\"data row2 col8\" >153</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col9\" class=\"data row2 col9\" >27</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col10\" class=\"data row2 col10\" >37</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col11\" class=\"data row2 col11\" >62</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col12\" class=\"data row2 col12\" >61</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row2_col13\" class=\"data row2 col13\" >20</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row3\" class=\"row_heading level0 row3\" >Athlete</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col0\" class=\"data row3 col0\" >20</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col1\" class=\"data row3 col1\" >1</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col2\" class=\"data row3 col2\" >769</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col3\" class=\"data row3 col3\" >2369</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col4\" class=\"data row3 col4\" >588</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col5\" class=\"data row3 col5\" >9</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col6\" class=\"data row3 col6\" >13</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col7\" class=\"data row3 col7\" >52</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col8\" class=\"data row3 col8\" >98</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col9\" class=\"data row3 col9\" >21</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col10\" class=\"data row3 col10\" >14</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col11\" class=\"data row3 col11\" >9</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col12\" class=\"data row3 col12\" >25</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row3_col13\" class=\"data row3 col13\" >12</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row4\" class=\"row_heading level0 row4\" >OfficeHolder</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col0\" class=\"data row4 col0\" >24</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col1\" class=\"data row4 col1\" >2</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col2\" class=\"data row4 col2\" >690</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col3\" class=\"data row4 col3\" >1200</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col4\" class=\"data row4 col4\" >1821</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col5\" class=\"data row4 col5\" >25</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col6\" class=\"data row4 col6\" >17</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col7\" class=\"data row4 col7\" >26</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col8\" class=\"data row4 col8\" >112</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col9\" class=\"data row4 col9\" >14</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col10\" class=\"data row4 col10\" >10</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col11\" class=\"data row4 col11\" >15</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col12\" class=\"data row4 col12\" >38</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row4_col13\" class=\"data row4 col13\" >6</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row5\" class=\"row_heading level0 row5\" >MeanOfTransportation</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col0\" class=\"data row5 col0\" >106</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col1\" class=\"data row5 col1\" >6</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col2\" class=\"data row5 col2\" >43</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col3\" class=\"data row5 col3\" >34</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col4\" class=\"data row5 col4\" >30</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col5\" class=\"data row5 col5\" >3541</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col6\" class=\"data row5 col6\" >19</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col7\" class=\"data row5 col7\" >26</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col8\" class=\"data row5 col8\" >46</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col9\" class=\"data row5 col9\" >36</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col10\" class=\"data row5 col10\" >19</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col11\" class=\"data row5 col11\" >59</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col12\" class=\"data row5 col12\" >26</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row5_col13\" class=\"data row5 col13\" >9</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row6\" class=\"row_heading level0 row6\" >Building</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col0\" class=\"data row6 col0\" >139</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col1\" class=\"data row6 col1\" >117</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col2\" class=\"data row6 col2\" >24</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col3\" class=\"data row6 col3\" >20</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col4\" class=\"data row6 col4\" >35</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col5\" class=\"data row6 col5\" >24</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col6\" class=\"data row6 col6\" >3331</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col7\" class=\"data row6 col7\" >90</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col8\" class=\"data row6 col8\" >56</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col9\" class=\"data row6 col9\" >15</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col10\" class=\"data row6 col10\" >19</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col11\" class=\"data row6 col11\" >49</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col12\" class=\"data row6 col12\" >46</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row6_col13\" class=\"data row6 col13\" >35</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row7\" class=\"row_heading level0 row7\" >NaturalPlace</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col0\" class=\"data row7 col0\" >82</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col1\" class=\"data row7 col1\" >2</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col2\" class=\"data row7 col2\" >34</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col3\" class=\"data row7 col3\" >49</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col4\" class=\"data row7 col4\" >35</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col5\" class=\"data row7 col5\" >15</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col6\" class=\"data row7 col6\" >50</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col7\" class=\"data row7 col7\" >3186</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col8\" class=\"data row7 col8\" >292</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col9\" class=\"data row7 col9\" >61</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col10\" class=\"data row7 col10\" >73</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col11\" class=\"data row7 col11\" >55</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col12\" class=\"data row7 col12\" >50</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row7_col13\" class=\"data row7 col13\" >16</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row8\" class=\"row_heading level0 row8\" >Village</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col0\" class=\"data row8 col0\" >71</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col1\" class=\"data row8 col1\" >3</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col2\" class=\"data row8 col2\" >44</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col3\" class=\"data row8 col3\" >91</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col4\" class=\"data row8 col4\" >89</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col5\" class=\"data row8 col5\" >12</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col6\" class=\"data row8 col6\" >27</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col7\" class=\"data row8 col7\" >111</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col8\" class=\"data row8 col8\" >3338</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col9\" class=\"data row8 col9\" >62</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col10\" class=\"data row8 col10\" >64</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col11\" class=\"data row8 col11\" >32</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col12\" class=\"data row8 col12\" >49</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row8_col13\" class=\"data row8 col13\" >7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row9\" class=\"row_heading level0 row9\" >Animal</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col0\" class=\"data row9 col0\" >44</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col1\" class=\"data row9 col1\" >1</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col2\" class=\"data row9 col2\" >10</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col3\" class=\"data row9 col3\" >27</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col4\" class=\"data row9 col4\" >11</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col5\" class=\"data row9 col5\" >13</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col6\" class=\"data row9 col6\" >10</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col7\" class=\"data row9 col7\" >55</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col8\" class=\"data row9 col8\" >103</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col9\" class=\"data row9 col9\" >2636</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col10\" class=\"data row9 col10\" >1000</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col11\" class=\"data row9 col11\" >35</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col12\" class=\"data row9 col12\" >39</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row9_col13\" class=\"data row9 col13\" >16</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row10\" class=\"row_heading level0 row10\" >Plant</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col0\" class=\"data row10 col0\" >39</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col1\" class=\"data row10 col1\" >0</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col2\" class=\"data row10 col2\" >7</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col3\" class=\"data row10 col3\" >7</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col4\" class=\"data row10 col4\" >4</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col5\" class=\"data row10 col5\" >3</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col6\" class=\"data row10 col6\" >6</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col7\" class=\"data row10 col7\" >23</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col8\" class=\"data row10 col8\" >87</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col9\" class=\"data row10 col9\" >643</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col10\" class=\"data row10 col10\" >3157</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col11\" class=\"data row10 col11\" >11</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col12\" class=\"data row10 col12\" >9</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row10_col13\" class=\"data row10 col13\" >4</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row11\" class=\"row_heading level0 row11\" >Album</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col0\" class=\"data row11 col0\" >218</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col1\" class=\"data row11 col1\" >6</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col2\" class=\"data row11 col2\" >53</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col3\" class=\"data row11 col3\" >50</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col4\" class=\"data row11 col4\" >19</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col5\" class=\"data row11 col5\" >51</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col6\" class=\"data row11 col6\" >71</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col7\" class=\"data row11 col7\" >67</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col8\" class=\"data row11 col8\" >87</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col9\" class=\"data row11 col9\" >81</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col10\" class=\"data row11 col10\" >51</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col11\" class=\"data row11 col11\" >2381</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col12\" class=\"data row11 col12\" >594</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row11_col13\" class=\"data row11 col13\" >271</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row12\" class=\"row_heading level0 row12\" >Film</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col0\" class=\"data row12 col0\" >136</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col1\" class=\"data row12 col1\" >11</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col2\" class=\"data row12 col2\" >79</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col3\" class=\"data row12 col3\" >73</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col4\" class=\"data row12 col4\" >83</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col5\" class=\"data row12 col5\" >32</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col6\" class=\"data row12 col6\" >77</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col7\" class=\"data row12 col7\" >82</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col8\" class=\"data row12 col8\" >302</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col9\" class=\"data row12 col9\" >81</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col10\" class=\"data row12 col10\" >72</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col11\" class=\"data row12 col11\" >937</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col12\" class=\"data row12 col12\" >1620</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row12_col13\" class=\"data row12 col13\" >415</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639level0_row13\" class=\"row_heading level0 row13\" >WrittenWork</th>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col0\" class=\"data row13 col0\" >204</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col1\" class=\"data row13 col1\" >16</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col2\" class=\"data row13 col2\" >35</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col3\" class=\"data row13 col3\" >34</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col4\" class=\"data row13 col4\" >27</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col5\" class=\"data row13 col5\" >30</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col6\" class=\"data row13 col6\" >59</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col7\" class=\"data row13 col7\" >50</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col8\" class=\"data row13 col8\" >98</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col9\" class=\"data row13 col9\" >66</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col10\" class=\"data row13 col10\" >36</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col11\" class=\"data row13 col11\" >534</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col12\" class=\"data row13 col12\" >608</td>\n",
       "                        <td id=\"T_d686cfce_dd72_11e9_9575_705a0f4b5639row13_col13\" class=\"data row13 col13\" >1082</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fc9d8bc7978>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "my_cmap = ListedColormap(sns.color_palette(\"BuGn\", 14).as_hex())\n",
    "s = cm_df.style.background_gradient(cmap=my_cmap)\n",
    "s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hal9002/.local/lib/python3.5/site-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type Net. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "net_path = 'trained_net.pt'\n",
    "torch.save(net, net_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name                                     Class                Score     \n",
      "\n",
      "Hello                                    Album                1.00      \n",
      "aaaaaaaaaaaaaaa                          Album                1.00      \n",
      "Streetbee                                Company              1.00      \n"
     ]
    }
   ],
   "source": [
    "### Load model\n",
    "net_path = 'trained_net.pt'\n",
    "net = torch.load(net_path)\n",
    "\n",
    "net.eval()\n",
    "\n",
    "input_names = ['Hello', 'aaaaaaaaaaaaaaa', 'Streetbee']\n",
    "message_embedding = sess.run(encoding_tensor, feed_dict={similarity_input_placeholder: input_names})\n",
    "\n",
    "# Uncomment 2 lines below for sample:\n",
    "# input_names = all_data['Name'].iloc[0:10]\n",
    "# message_embedding = all_data['Name_emb'].iloc[0:10]\n",
    "\n",
    "tensor_input = torch.Tensor(message_embedding).to(device)\n",
    "tensor_input\n",
    "predicted_class_score = [(int(torch.argmax(x)), max(x)) for x in net(tensor_input)]\n",
    "\n",
    "print('{0:<40} {1:<20} {2:<10}'.format('Name', 'Class', 'Score'))\n",
    "print()\n",
    "for enum, x in enumerate(predicted_class_score):\n",
    "    print('{0:<40} {1:<20} {2:<10.2f}'.format(input_names[enum], class_dict[x[0]], x[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}