BoW_ML_Experiments.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "BoW_ML_Experiments.ipynb",
   "provenance": [],
   "collapsed_sections": [],
   "authorship_tag": "ABX9TyPV3YLLepM5AH7vdYDz/ocZ"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "id": "YDHCjaH5QrDk",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406523706,
     "user_tz": -120,
     "elapsed": 2178,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "787da346-13dc-4f00-e01b-63e478d67258"
   },
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive/', force_remount=True)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive/\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "_t6gg1ZLRYxS",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import sys\n",
    "projectDir = \"/content/drive/My Drive/code/git/tweepfake_deepfake_text_detection\"\n",
    "sys.path.insert(0, projectDir)\n",
    "\n",
    "resultsDir = projectDir+\"/data/results\"\n",
    "random_state = 523 # Fixed seed for replicability of randomic operations."
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "V3qW3001RonB",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406526815,
     "user_tz": -120,
     "elapsed": 5272,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "27f06aee-dbbd-4864-a29e-4fc093ff3f4f"
   },
   "source": [
    "!pip install pandas"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (1.0.5)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from pandas) (1.18.5)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas) (1.12.0)\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "lLsNhiofRy5r",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from DataHandler import DataHandler\n",
    "\n",
    "csvTrainDataset = projectDir+\"/data/splits/train.csv\"\n",
    "csvValDataset = projectDir+\"/data/splits/validation.csv\"\n",
    "csvTestDataset = projectDir+\"/data/splits/test.csv\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "dh = DataHandler()\n",
    "dfTrain = dh.readCSVData(csvTrainDataset)\n",
    "dfVal = dh.readCSVData(csvValDataset)\n",
    "dfTest = dh.readCSVData(csvTestDataset)\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mBCDg773SZKW",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406527635,
     "user_tz": -120,
     "elapsed": 6077,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "cddd7b36-dc2e-48cc-f16e-ae2d9d702c36"
   },
   "source": [
    "dfTrain"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>N_prog</th>\n",
       "      <th>user_id</th>\n",
       "      <th>status_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>source</th>\n",
       "      <th>display_text_width</th>\n",
       "      <th>reply_to_status_id</th>\n",
       "      <th>reply_to_user_id</th>\n",
       "      <th>reply_to_screen_name</th>\n",
       "      <th>is_quote</th>\n",
       "      <th>is_retweet</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>quote_count</th>\n",
       "      <th>reply_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>symbols</th>\n",
       "      <th>urls_url</th>\n",
       "      <th>urls_t.co</th>\n",
       "      <th>urls_expanded_url</th>\n",
       "      <th>media_url</th>\n",
       "      <th>media_t.co</th>\n",
       "      <th>media_expanded_url</th>\n",
       "      <th>media_type</th>\n",
       "      <th>ext_media_url</th>\n",
       "      <th>ext_media_t.co</th>\n",
       "      <th>ext_media_expanded_url</th>\n",
       "      <th>ext_media_type</th>\n",
       "      <th>mentions_user_id</th>\n",
       "      <th>mentions_screen_name</th>\n",
       "      <th>lang</th>\n",
       "      <th>quoted_status_id</th>\n",
       "      <th>quoted_text</th>\n",
       "      <th>quoted_created_at</th>\n",
       "      <th>quoted_source</th>\n",
       "      <th>quoted_favorite_count</th>\n",
       "      <th>quoted_retweet_count</th>\n",
       "      <th>quoted_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>place_url</th>\n",
       "      <th>place_name</th>\n",
       "      <th>place_full_name</th>\n",
       "      <th>place_type</th>\n",
       "      <th>country</th>\n",
       "      <th>country_code</th>\n",
       "      <th>geo_coords</th>\n",
       "      <th>coords_coords</th>\n",
       "      <th>bbox_coords</th>\n",
       "      <th>status_url</th>\n",
       "      <th>name</th>\n",
       "      <th>location</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>protected</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>listed_count</th>\n",
       "      <th>statuses_count</th>\n",
       "      <th>favourites_count</th>\n",
       "      <th>account_created_at</th>\n",
       "      <th>verified</th>\n",
       "      <th>profile_url</th>\n",
       "      <th>profile_expanded_url</th>\n",
       "      <th>account_lang</th>\n",
       "      <th>profile_banner_url</th>\n",
       "      <th>profile_background_url</th>\n",
       "      <th>profile_image_url</th>\n",
       "      <th>created</th>\n",
       "      <th>data.tweet</th>\n",
       "      <th>ora.tweet</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>mention_TF</th>\n",
       "      <th>hashtag_TF</th>\n",
       "      <th>url_TF</th>\n",
       "      <th>n.words</th>\n",
       "      <th>punct</th>\n",
       "      <th>account.type</th>\n",
       "      <th>finetuning_source</th>\n",
       "      <th>class_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14692</td>\n",
       "      <td>x1110407881030017024</td>\n",
       "      <td>x1208265880146046976</td>\n",
       "      <td>2019-12-21 06:00:08</td>\n",
       "      <td>imranyebot</td>\n",
       "      <td>YEA now that note GOOD</td>\n",
       "      <td>imranye-twitter-bot</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/imranyebot/status/12082658...</td>\n",
       "      <td>imranyebot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@imranye in robot form. tweets every hour gene...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>7467</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-03-26 05:07:23</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1110408254...</td>\n",
       "      <td>2019-12-21 06:00:08</td>\n",
       "      <td>2019-12-21</td>\n",
       "      <td>06:00:08</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>bot</td>\n",
       "      <td>imranye</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22336</td>\n",
       "      <td>x3171109449</td>\n",
       "      <td>x1091463908118941696</td>\n",
       "      <td>2019-02-01 22:30:48</td>\n",
       "      <td>zawvrk</td>\n",
       "      <td>Listen to This Charming Man by The Smiths  htt...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>66</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>soundcloud.com/thesmiths/this…</td>\n",
       "      <td>https://t.co/r12OIXkfUO</td>\n",
       "      <td>https://soundcloud.com/thesmiths/this-charming...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/zawvrk/status/109146390811...</td>\n",
       "      <td>zawar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>gold experience</td>\n",
       "      <td>https://t.co/J60plTSsKt</td>\n",
       "      <td>False</td>\n",
       "      <td>558</td>\n",
       "      <td>274</td>\n",
       "      <td>8</td>\n",
       "      <td>12216</td>\n",
       "      <td>23633</td>\n",
       "      <td>2015-04-15 19:50:16</td>\n",
       "      <td>False</td>\n",
       "      <td>https://t.co/J60plTSsKt</td>\n",
       "      <td>https://www.instagram.com/zawvr</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pbs.twimg.com/profile_banners/31711094...</td>\n",
       "      <td>http://abs.twimg.com/images/themes/theme1/bg.png</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1229957200...</td>\n",
       "      <td>2019-02-01 22:30:48</td>\n",
       "      <td>2019-02-01</td>\n",
       "      <td>22:30:48</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>human</td>\n",
       "      <td>original</td>\n",
       "      <td>human</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24087</td>\n",
       "      <td>x1110686081341632512</td>\n",
       "      <td>x1199055191028293633</td>\n",
       "      <td>2019-11-25 20:00:09</td>\n",
       "      <td>zawarbot</td>\n",
       "      <td>wish i can i would be seeing other hoes on the...</td>\n",
       "      <td>zawar-twitter-bot</td>\n",
       "      <td>57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/zawarbot/status/1199055191...</td>\n",
       "      <td>zawarbot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I’m an AI Bot that tries to tweet like @zawvrk...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>4054</td>\n",
       "      <td>9</td>\n",
       "      <td>2019-03-26 23:32:52</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://abs.twimg.com/images/themes/theme1/bg.png</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1185589257...</td>\n",
       "      <td>2019-11-25 20:00:09</td>\n",
       "      <td>2019-11-25</td>\n",
       "      <td>20:00:09</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>bot</td>\n",
       "      <td>zawvrk</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7584</td>\n",
       "      <td>x1110307772783124480</td>\n",
       "      <td>x1214698264701722626</td>\n",
       "      <td>2020-01-08 00:00:08</td>\n",
       "      <td>ahadsheriffbot</td>\n",
       "      <td>The decade in the significantly easier schedul...</td>\n",
       "      <td>ahadsheriff-bot</td>\n",
       "      <td>63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/ahadsheriffbot/status/1214...</td>\n",
       "      <td>Robot Ahad Sheriff</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@ahadsheriff in robot form. tweets every hour ...</td>\n",
       "      <td>https://t.co/2G243YsNOt</td>\n",
       "      <td>False</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7946</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-03-25 22:29:36</td>\n",
       "      <td>False</td>\n",
       "      <td>https://t.co/2G243YsNOt</td>\n",
       "      <td>https://ahadsheriff.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://abs.twimg.com/images/themes/theme1/bg.png</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1110311752...</td>\n",
       "      <td>2020-01-08 00:00:08</td>\n",
       "      <td>2020-01-08</td>\n",
       "      <td>00:00:08</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>bot</td>\n",
       "      <td>ahadsheriff</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19654</td>\n",
       "      <td>x979586167405363200</td>\n",
       "      <td>x1209229478934695937</td>\n",
       "      <td>2019-12-23 21:49:08</td>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Theim class=\\\"alignnone size-full wp-image-60...</td>\n",
       "      <td>kevinhooke-ml-bot</td>\n",
       "      <td>180</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kevinhooke.com/wp-content/upl…</td>\n",
       "      <td>https://t.co/LiAsQsbs99</td>\n",
       "      <td>https://www.kevinhooke.com/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/kevinhookebot/status/12092...</td>\n",
       "      <td>Kevin Hooke Bot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I'm an AWS Lambda generating content from a Re...</td>\n",
       "      <td>https://t.co/gTSHg4mXqN</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6608</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-03-30 05:08:36</td>\n",
       "      <td>False</td>\n",
       "      <td>https://t.co/gTSHg4mXqN</td>\n",
       "      <td>https://www.kevinhooke.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://abs.twimg.com/sticky/default_profile_im...</td>\n",
       "      <td>2019-12-23 21:49:08</td>\n",
       "      <td>2019-12-23</td>\n",
       "      <td>21:49:08</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>30</td>\n",
       "      <td>27</td>\n",
       "      <td>bot</td>\n",
       "      <td>kevinhooke</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20707</th>\n",
       "      <td>1538</td>\n",
       "      <td>x1197916267975335939</td>\n",
       "      <td>x1204245032917700608</td>\n",
       "      <td>2019-12-10 03:42:44</td>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Met on the Abversion of our science for the co...</td>\n",
       "      <td>AI_Narendra_Modi</td>\n",
       "      <td>103</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>life.Om</td>\n",
       "      <td>https://t.co/kl5Dfl0y18</td>\n",
       "      <td>http://life.Om</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/AINarendraModi/status/1204...</td>\n",
       "      <td>AI Narendra Modi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Narendra Modi but with AI\\r\\nBot commands - ht...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1625</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-11-22 16:34:45</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pbs.twimg.com/profile_banners/11979162...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1197916520...</td>\n",
       "      <td>2019-12-10 03:42:44</td>\n",
       "      <td>2019-12-10</td>\n",
       "      <td>03:42:44</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>bot</td>\n",
       "      <td>narendramodi</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20708</th>\n",
       "      <td>1885</td>\n",
       "      <td>x1197916267975335939</td>\n",
       "      <td>x1207011474243309570</td>\n",
       "      <td>2019-12-17 18:55:35</td>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Land for their during the opportunity to the p...</td>\n",
       "      <td>AI_Narendra_Modi</td>\n",
       "      <td>85</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/AINarendraModi/status/1207...</td>\n",
       "      <td>AI Narendra Modi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Narendra Modi but with AI\\r\\nBot commands - ht...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1625</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-11-22 16:34:45</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pbs.twimg.com/profile_banners/11979162...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1197916520...</td>\n",
       "      <td>2019-12-17 18:55:35</td>\n",
       "      <td>2019-12-17</td>\n",
       "      <td>18:55:35</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>bot</td>\n",
       "      <td>narendramodi</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20709</th>\n",
       "      <td>3930</td>\n",
       "      <td>x705113652471439361</td>\n",
       "      <td>x715558455285837824</td>\n",
       "      <td>2016-03-31 15:16:44</td>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>@TayandYou doesn't have a clue. You're right. ...</td>\n",
       "      <td>Twitter Web Client</td>\n",
       "      <td>133</td>\n",
       "      <td>x714838749860012032</td>\n",
       "      <td>x78324623</td>\n",
       "      <td>eldiariodedross</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>x4531940473 x78324623</td>\n",
       "      <td>TayandYou eldiariodedross</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/DeepDrumpf/status/71555845...</td>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I'm a Neural Network trained on Trump's transc...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>25401</td>\n",
       "      <td>7</td>\n",
       "      <td>296</td>\n",
       "      <td>286</td>\n",
       "      <td>19</td>\n",
       "      <td>2016-03-02 19:32:49</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pbs.twimg.com/profile_banners/70511365...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/7054647353...</td>\n",
       "      <td>2016-03-31 15:16:44</td>\n",
       "      <td>2016-03-31</td>\n",
       "      <td>15:16:44</td>\n",
       "      <td>reply</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>26</td>\n",
       "      <td>10</td>\n",
       "      <td>bot</td>\n",
       "      <td>realDonaldTrump</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20710</th>\n",
       "      <td>729</td>\n",
       "      <td>x262794965</td>\n",
       "      <td>x812868913239199745</td>\n",
       "      <td>2016-12-25 03:53:45</td>\n",
       "      <td>jaden</td>\n",
       "      <td>Me And My Bestie https://t.co/vPq2iDkWZm</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7265</td>\n",
       "      <td>1071</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://pbs.twimg.com/media/C0fj97lUQAAT-dC.jpg</td>\n",
       "      <td>https://t.co/vPq2iDkWZm</td>\n",
       "      <td>https://twitter.com/officialjaden/status/81286...</td>\n",
       "      <td>photo</td>\n",
       "      <td>http://pbs.twimg.com/media/C0fj97lUQAAT-dC.jpg</td>\n",
       "      <td>https://t.co/vPq2iDkWZm</td>\n",
       "      <td>https://twitter.com/officialjaden/status/81286...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/jaden/status/8128689132391...</td>\n",
       "      <td>Jaden</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;U+0001F308&gt; Boy</td>\n",
       "      <td>https://t.co/2dZJOThjAQ</td>\n",
       "      <td>False</td>\n",
       "      <td>8142789</td>\n",
       "      <td>188</td>\n",
       "      <td>14893</td>\n",
       "      <td>5669</td>\n",
       "      <td>1844</td>\n",
       "      <td>2011-03-08 19:41:06</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/2dZJOThjAQ</td>\n",
       "      <td>https://jaden.link/erys</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pbs.twimg.com/profile_banners/26279496...</td>\n",
       "      <td>http://abs.twimg.com/images/themes/theme1/bg.png</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1146090110...</td>\n",
       "      <td>2016-12-25 03:53:45</td>\n",
       "      <td>2016-12-25</td>\n",
       "      <td>03:53:45</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>human</td>\n",
       "      <td>original</td>\n",
       "      <td>human</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20711</th>\n",
       "      <td>19366</td>\n",
       "      <td>x979586167405363200</td>\n",
       "      <td>x1194703766303338496</td>\n",
       "      <td>2019-11-13 19:49:09</td>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Thead has a generate existing the Sparching f...</td>\n",
       "      <td>kevinhooke-ml-bot</td>\n",
       "      <td>205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA</td>\n",
       "      <td>NA NA NA NA NA NA NA NA</td>\n",
       "      <td>https://twitter.com/kevinhookebot/status/11947...</td>\n",
       "      <td>Kevin Hooke Bot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I'm an AWS Lambda generating content from a Re...</td>\n",
       "      <td>https://t.co/gTSHg4mXqN</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6608</td>\n",
       "      <td>0</td>\n",
       "      <td>2018-03-30 05:08:36</td>\n",
       "      <td>False</td>\n",
       "      <td>https://t.co/gTSHg4mXqN</td>\n",
       "      <td>https://www.kevinhooke.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://abs.twimg.com/sticky/default_profile_im...</td>\n",
       "      <td>2019-11-13 19:49:09</td>\n",
       "      <td>2019-11-13</td>\n",
       "      <td>19:49:09</td>\n",
       "      <td>tweet</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "      <td>bot</td>\n",
       "      <td>kevinhooke</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20712 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      N_prog               user_id  ... finetuning_source class_type\n",
       "0      14692  x1110407881030017024  ...           imranye     others\n",
       "1      22336           x3171109449  ...          original      human\n",
       "2      24087  x1110686081341632512  ...            zawvrk     others\n",
       "3       7584  x1110307772783124480  ...       ahadsheriff     others\n",
       "4      19654   x979586167405363200  ...        kevinhooke        rnn\n",
       "...      ...                   ...  ...               ...        ...\n",
       "20707   1538  x1197916267975335939  ...      narendramodi        rnn\n",
       "20708   1885  x1197916267975335939  ...      narendramodi        rnn\n",
       "20709   3930   x705113652471439361  ...   realDonaldTrump        rnn\n",
       "20710    729            x262794965  ...          original      human\n",
       "20711  19366   x979586167405363200  ...        kevinhooke        rnn\n",
       "\n",
       "[20712 rows x 103 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 5
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "c0oedq8-W7u2",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "# Select interesting columns for this study.\n",
    "dfTrainDataset = dfTrain[[\"screen_name\", \"text\", \"account.type\", \"class_type\"]]\n",
    "dfValDataset = dfVal[[\"screen_name\", \"text\", \"account.type\", \"class_type\"]]\n",
    "dfTestDataset = dfTest[[\"screen_name\", \"text\", \"account.type\", \"class_type\"]]"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "qTeVpVc6I9Lb",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406527636,
     "user_tz": -120,
     "elapsed": 6062,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "7b8ac24a-9cf0-4676-d426-58b9fa3f3d7d"
   },
   "source": [
    "dfTrainDataset"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>account.type</th>\n",
       "      <th>class_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>imranyebot</td>\n",
       "      <td>YEA now that note GOOD</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zawvrk</td>\n",
       "      <td>Listen to This Charming Man by The Smiths  htt...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zawarbot</td>\n",
       "      <td>wish i can i would be seeing other hoes on the...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ahadsheriffbot</td>\n",
       "      <td>The decade in the significantly easier schedul...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Theim class=\\\"alignnone size-full wp-image-60...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20707</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Met on the Abversion of our science for the co...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20708</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Land for their during the opportunity to the p...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20709</th>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>@TayandYou doesn't have a clue. You're right. ...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20710</th>\n",
       "      <td>jaden</td>\n",
       "      <td>Me And My Bestie https://t.co/vPq2iDkWZm</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20711</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Thead has a generate existing the Sparching f...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20712 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          screen_name  ... class_type\n",
       "0          imranyebot  ...     others\n",
       "1              zawvrk  ...      human\n",
       "2            zawarbot  ...     others\n",
       "3      ahadsheriffbot  ...     others\n",
       "4       kevinhookebot  ...        rnn\n",
       "...               ...  ...        ...\n",
       "20707  AINarendraModi  ...        rnn\n",
       "20708  AINarendraModi  ...        rnn\n",
       "20709      DeepDrumpf  ...        rnn\n",
       "20710           jaden  ...      human\n",
       "20711   kevinhookebot  ...        rnn\n",
       "\n",
       "[20712 rows x 4 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 7
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "gWNVRnyfJKKH",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534142,
     "user_tz": -120,
     "elapsed": 12560,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "476cd1ec-39f8-4cb3-de83-6f823e5c7b04"
   },
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "import re\n",
    "\n",
    "def tokenizeTweets(df):\n",
    "  tweet_tokenizer = TweetTokenizer()\n",
    "  tweet_tokens = []\n",
    "  for index, row in df.iterrows():\n",
    "    text = row[\"text\"]\n",
    "    tokenized = tweet_tokenizer.tokenize(text)\n",
    "    tokenized = [t.lower() for t in tokenized]\n",
    "    tokenized = [t for t in tokenized if not t.startswith(\"#\")]\n",
    "    tokenized_mod = []\n",
    "    for t in tokenized:\n",
    "      if t.startswith(\"@\"):\n",
    "        if len(t) > 1:\n",
    "          tokenized_mod.append(\"__user_mention__\")\n",
    "        else:\n",
    "          tokenized_mod.append(\"__invalid_user_mention__\")\n",
    "      else:\n",
    "        tokenized_mod.append(t)\n",
    "    tokenized = tokenized_mod\n",
    "    allText = \" \".join(tokenized)\n",
    "    allText = re.sub(r'http\\S+', '__url__', allText)\n",
    "    tokenized = allText.split()\n",
    "    tweet_tokens.append(tokenized)\n",
    "\n",
    "  dfRet = df.copy()\n",
    "  dfRet[\"text_tokenized\"] = tweet_tokens\n",
    "  return dfRet\n",
    "\n",
    "\n",
    "dfTrainDatasetTokenized = tokenizeTweets(dfTrainDataset)\n",
    "dfValDatasetTokenized = tokenizeTweets(dfValDataset)\n",
    "dfTestDatasetTokenized = tokenizeTweets(dfTestDataset)\n",
    "dfTrainDatasetTokenized"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>account.type</th>\n",
       "      <th>class_type</th>\n",
       "      <th>text_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>imranyebot</td>\n",
       "      <td>YEA now that note GOOD</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[yea, now, that, note, good]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zawvrk</td>\n",
       "      <td>Listen to This Charming Man by The Smiths  htt...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[listen, to, this, charming, man, by, the, smi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zawarbot</td>\n",
       "      <td>wish i can i would be seeing other hoes on the...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[wish, i, can, i, would, be, seeing, other, ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ahadsheriffbot</td>\n",
       "      <td>The decade in the significantly easier schedul...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[the, decade, in, the, significantly, easier, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Theim class=\\\"alignnone size-full wp-image-60...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[\", theim, class, =\\, \", alignnone, size-full,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20707</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Met on the Abversion of our science for the co...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[met, on, the, abversion, of, our, science, fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20708</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Land for their during the opportunity to the p...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[land, for, their, during, the, opportunity, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20709</th>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>@TayandYou doesn't have a clue. You're right. ...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[__user_mention__, doesn't, have, a, clue, ., ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20710</th>\n",
       "      <td>jaden</td>\n",
       "      <td>Me And My Bestie https://t.co/vPq2iDkWZm</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[me, and, my, bestie, __url__]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20711</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Thead has a generate existing the Sparching f...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[\", thead, has, a, generate, existing, the, sp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20712 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          screen_name  ...                                     text_tokenized\n",
       "0          imranyebot  ...                       [yea, now, that, note, good]\n",
       "1              zawvrk  ...  [listen, to, this, charming, man, by, the, smi...\n",
       "2            zawarbot  ...  [wish, i, can, i, would, be, seeing, other, ho...\n",
       "3      ahadsheriffbot  ...  [the, decade, in, the, significantly, easier, ...\n",
       "4       kevinhookebot  ...  [\", theim, class, =\\, \", alignnone, size-full,...\n",
       "...               ...  ...                                                ...\n",
       "20707  AINarendraModi  ...  [met, on, the, abversion, of, our, science, fo...\n",
       "20708  AINarendraModi  ...  [land, for, their, during, the, opportunity, t...\n",
       "20709      DeepDrumpf  ...  [__user_mention__, doesn't, have, a, clue, ., ...\n",
       "20710           jaden  ...                     [me, and, my, bestie, __url__]\n",
       "20711   kevinhookebot  ...  [\", thead, has, a, generate, existing, the, sp...\n",
       "\n",
       "[20712 rows x 5 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 8
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "1tHTftwVR57f",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "X_train_all = dfTrainDatasetTokenized\n",
    "X_val_all = dfValDatasetTokenized\n",
    "X_test_all = dfTestDatasetTokenized\n",
    "dictLabels = {\"human\":0, \"bot\":1}\n",
    "dictLabelsReverse = {0:\"human\", 1: \"bot\"}\n",
    "y_train = dfTrainDatasetTokenized[\"account.type\"].apply(lambda x: dictLabels[x])\n",
    "y_val = dfValDatasetTokenized[\"account.type\"].apply(lambda x: dictLabels[x])\n",
    "y_test = dfTestDatasetTokenized[\"account.type\"].apply(lambda x: dictLabels[x])"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "lxGEqX35jpEL",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534144,
     "user_tz": -120,
     "elapsed": 12548,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "a3ac390c-7b05-4b2f-bb2e-533758077337"
   },
   "source": [
    "y_val"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0       0\n",
       "1       0\n",
       "2       0\n",
       "3       0\n",
       "4       1\n",
       "       ..\n",
       "2297    1\n",
       "2298    0\n",
       "2299    0\n",
       "2300    1\n",
       "2301    1\n",
       "Name: account.type, Length: 2302, dtype: int64"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 10
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "XpvyC9KDzm47",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534145,
     "user_tz": -120,
     "elapsed": 12541,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "1609a7a9-3e60-4237-9cd1-8ccbfb13a7e5"
   },
   "source": [
    "X_train_all"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>account.type</th>\n",
       "      <th>class_type</th>\n",
       "      <th>text_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>imranyebot</td>\n",
       "      <td>YEA now that note GOOD</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[yea, now, that, note, good]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zawvrk</td>\n",
       "      <td>Listen to This Charming Man by The Smiths  htt...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[listen, to, this, charming, man, by, the, smi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zawarbot</td>\n",
       "      <td>wish i can i would be seeing other hoes on the...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[wish, i, can, i, would, be, seeing, other, ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ahadsheriffbot</td>\n",
       "      <td>The decade in the significantly easier schedul...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[the, decade, in, the, significantly, easier, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Theim class=\\\"alignnone size-full wp-image-60...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[\", theim, class, =\\, \", alignnone, size-full,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20707</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Met on the Abversion of our science for the co...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[met, on, the, abversion, of, our, science, fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20708</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Land for their during the opportunity to the p...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[land, for, their, during, the, opportunity, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20709</th>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>@TayandYou doesn't have a clue. You're right. ...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[__user_mention__, doesn't, have, a, clue, ., ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20710</th>\n",
       "      <td>jaden</td>\n",
       "      <td>Me And My Bestie https://t.co/vPq2iDkWZm</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[me, and, my, bestie, __url__]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20711</th>\n",
       "      <td>kevinhookebot</td>\n",
       "      <td>\"Thead has a generate existing the Sparching f...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[\", thead, has, a, generate, existing, the, sp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20712 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          screen_name  ...                                     text_tokenized\n",
       "0          imranyebot  ...                       [yea, now, that, note, good]\n",
       "1              zawvrk  ...  [listen, to, this, charming, man, by, the, smi...\n",
       "2            zawarbot  ...  [wish, i, can, i, would, be, seeing, other, ho...\n",
       "3      ahadsheriffbot  ...  [the, decade, in, the, significantly, easier, ...\n",
       "4       kevinhookebot  ...  [\", theim, class, =\\, \", alignnone, size-full,...\n",
       "...               ...  ...                                                ...\n",
       "20707  AINarendraModi  ...  [met, on, the, abversion, of, our, science, fo...\n",
       "20708  AINarendraModi  ...  [land, for, their, during, the, opportunity, t...\n",
       "20709      DeepDrumpf  ...  [__user_mention__, doesn't, have, a, clue, ., ...\n",
       "20710           jaden  ...                     [me, and, my, bestie, __url__]\n",
       "20711   kevinhookebot  ...  [\", thead, has, a, generate, existing, the, sp...\n",
       "\n",
       "[20712 rows x 5 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 11
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ijLSkPTm0K6f",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534145,
     "user_tz": -120,
     "elapsed": 12534,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "42c214c2-78f7-4faa-c892-304bbe3f73a3"
   },
   "source": [
    "X_val_all"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>account.type</th>\n",
       "      <th>class_type</th>\n",
       "      <th>text_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ahadsheriff</td>\n",
       "      <td>TIGHT, TIGHT, TIGHT, YEAH!!! https://t.co/wj3n...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[tight, ,, tight, ,, tight, ,, yeah, !, !, !, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>narendramodi</td>\n",
       "      <td>India has millennia old relations with Oman. W...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[india, has, millennia, old, relations, with, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>jaden</td>\n",
       "      <td>Anxious Teenagers</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[anxious, teenagers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JustinTrudeau</td>\n",
       "      <td>Our top priority is keeping Canadians safe. Wi...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[our, top, priority, is, keeping, canadians, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>imranyebot</td>\n",
       "      <td>nah bro You’re taking sis so much I’m just a g...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[nah, bro, you, ’, re, taking, sis, so, much, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2297</th>\n",
       "      <td>DeepDrumpf</td>\n",
       "      <td>You're going to be even prouder when we don't ...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[you're, going, to, be, even, prouder, when, w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2298</th>\n",
       "      <td>jaden</td>\n",
       "      <td>https://t.co/10XkzXDBCf https://t.co/cIUIYWEB45</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[__url__, __url__]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2299</th>\n",
       "      <td>ahadsheriff</td>\n",
       "      <td>2. “Once you take the place of the people who ...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[2, ., “, once, you, take, the, place, of, the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2300</th>\n",
       "      <td>imranyebot</td>\n",
       "      <td>black will be like a company with them need so...</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[black, will, be, like, a, company, with, them...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2301</th>\n",
       "      <td>GenePark_GPT2</td>\n",
       "      <td>Guys, I hate Facebook. And this Facebook ad ca...</td>\n",
       "      <td>bot</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>[guys, ,, i, hate, facebook, ., and, this, fac...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2302 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        screen_name  ...                                     text_tokenized\n",
       "0       ahadsheriff  ...  [tight, ,, tight, ,, tight, ,, yeah, !, !, !, ...\n",
       "1      narendramodi  ...  [india, has, millennia, old, relations, with, ...\n",
       "2             jaden  ...                               [anxious, teenagers]\n",
       "3     JustinTrudeau  ...  [our, top, priority, is, keeping, canadians, s...\n",
       "4        imranyebot  ...  [nah, bro, you, ’, re, taking, sis, so, much, ...\n",
       "...             ...  ...                                                ...\n",
       "2297     DeepDrumpf  ...  [you're, going, to, be, even, prouder, when, w...\n",
       "2298          jaden  ...                                 [__url__, __url__]\n",
       "2299    ahadsheriff  ...  [2, ., “, once, you, take, the, place, of, the...\n",
       "2300     imranyebot  ...  [black, will, be, like, a, company, with, them...\n",
       "2301  GenePark_GPT2  ...  [guys, ,, i, hate, facebook, ., and, this, fac...\n",
       "\n",
       "[2302 rows x 5 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 12
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "V3WYP4xQ-Kvz",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534146,
     "user_tz": -120,
     "elapsed": 12527,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "1ed9fc1d-def3-461a-e8ed-82db19994662"
   },
   "source": [
    "X_test_all"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>account.type</th>\n",
       "      <th>class_type</th>\n",
       "      <th>text_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zawvrk</td>\n",
       "      <td>justin timberlake really one of the goats if y...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[justin, timberlake, really, one, of, the, goa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>narendramodi</td>\n",
       "      <td>Thank you @PMBhutan for your gracious prayers ...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[thank, you, __user_mention__, for, your, grac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ahadsheriff</td>\n",
       "      <td>Theory: the number of red lights you will hit ...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[theory, :, the, number, of, red, lights, you,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AINarendraModi</td>\n",
       "      <td>Respects on the Upt of the I good with the peo...</td>\n",
       "      <td>bot</td>\n",
       "      <td>rnn</td>\n",
       "      <td>[respects, on, the, upt, of, the, i, good, wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kevinhooke</td>\n",
       "      <td>Might give the BASIC #10Liner game contest ano...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[might, give, the, basic, game, contest, anoth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2553</th>\n",
       "      <td>ahadsheriffbot</td>\n",
       "      <td>“The best kept secret</td>\n",
       "      <td>bot</td>\n",
       "      <td>others</td>\n",
       "      <td>[“, the, best, kept, secret]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2554</th>\n",
       "      <td>kevinhooke</td>\n",
       "      <td>Love the Choose your own adventure style of th...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[love, the, choose, your, own, adventure, styl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2555</th>\n",
       "      <td>dril_gpt2</td>\n",
       "      <td>JOIN OUR TEAM: Sneezing</td>\n",
       "      <td>bot</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>[join, our, team, :, sneezing]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2556</th>\n",
       "      <td>kevinhooke</td>\n",
       "      <td>These deeply discounted 256GB SanDisk flash dr...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[these, deeply, discounted, 256gb, sandisk, fl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2557</th>\n",
       "      <td>zawvrk</td>\n",
       "      <td>@deleonfc6 diamond pearl and platinum were hon...</td>\n",
       "      <td>human</td>\n",
       "      <td>human</td>\n",
       "      <td>[__user_mention__, diamond, pearl, and, platin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2558 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         screen_name  ...                                     text_tokenized\n",
       "0             zawvrk  ...  [justin, timberlake, really, one, of, the, goa...\n",
       "1       narendramodi  ...  [thank, you, __user_mention__, for, your, grac...\n",
       "2        ahadsheriff  ...  [theory, :, the, number, of, red, lights, you,...\n",
       "3     AINarendraModi  ...  [respects, on, the, upt, of, the, i, good, wit...\n",
       "4         kevinhooke  ...  [might, give, the, basic, game, contest, anoth...\n",
       "...              ...  ...                                                ...\n",
       "2553  ahadsheriffbot  ...                       [“, the, best, kept, secret]\n",
       "2554      kevinhooke  ...  [love, the, choose, your, own, adventure, styl...\n",
       "2555       dril_gpt2  ...                     [join, our, team, :, sneezing]\n",
       "2556      kevinhooke  ...  [these, deeply, discounted, 256gb, sandisk, fl...\n",
       "2557          zawvrk  ...  [__user_mention__, diamond, pearl, and, platin...\n",
       "\n",
       "[2558 rows x 5 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 13
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "vea0LTUMg5zE",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "X_train = X_train_all[\"text_tokenized\"].tolist()\n",
    "X_train = [\" \".join(i) for i in X_train]\n",
    "\n",
    "\n",
    "X_val = X_val_all[\"text_tokenized\"].tolist()\n",
    "X_val = [\" \".join(i) for i in X_val]\n",
    "\n",
    "X_test = X_test_all[\"text_tokenized\"].tolist()\n",
    "X_test = [\" \".join(i) for i in X_test]"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Tzrz4NKzJd9N",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534147,
     "user_tz": -120,
     "elapsed": 12515,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "814947ee-c4a2-40a7-af97-badf539fe165"
   },
   "source": [
    "len(X_train)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "20712"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 15
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "PUMg622lKP-6",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534147,
     "user_tz": -120,
     "elapsed": 12500,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "106d9aa5-2279-4edc-8823-e726fda095a1"
   },
   "source": [
    "len(X_val)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2302"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 16
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "YSC4VZW6LGsY",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534148,
     "user_tz": -120,
     "elapsed": 12492,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "e0c4efba-f5ff-47a7-913e-971854a5e23b"
   },
   "source": [
    "len(X_test)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2558"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 17
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "3hD04InuRY-F",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "\n",
    "vect = TfidfVectorizer(ngram_range=(1, 1), max_features=25000, \n",
    "                       dtype=np.float32)\n",
    "train_features = vect.fit_transform(X_train)\n",
    "valid_features = vect.transform(X_val)\n",
    "test_features = vect.transform(X_test)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "h3IkZvjmSyNQ",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406534519,
     "user_tz": -120,
     "elapsed": 12844,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "cfc43af9-1939-49cb-d19a-11ba45e2e54a"
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "first_vector_tfidfvectorizer = train_features[0]\n",
    "dfVec = pd.DataFrame(first_vector_tfidfvectorizer.T.todense(), index=vect.get_feature_names(), columns=[\"tfidf\"])\n",
    "dfVec.sort_values(by=[\"tfidf\"],ascending=False)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>note</th>\n",
       "      <td>0.599652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yea</th>\n",
       "      <td>0.591628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>now</th>\n",
       "      <td>0.350527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>good</th>\n",
       "      <td>0.341828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>that</th>\n",
       "      <td>0.225116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edish</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edict</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edgy</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edges</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>édouard</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            tfidf\n",
       "note     0.599652\n",
       "yea      0.591628\n",
       "now      0.350527\n",
       "good     0.341828\n",
       "that     0.225116\n",
       "...           ...\n",
       "edish    0.000000\n",
       "edict    0.000000\n",
       "edgy     0.000000\n",
       "edges    0.000000\n",
       "édouard  0.000000\n",
       "\n",
       "[25000 rows x 1 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 19
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "xDAhMIgmpxTs",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "train_labels = y_train.tolist()\n",
    "val_labels = y_val.tolist()\n",
    "test_labels = y_test.tolist()"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WyBJOn97WM9a",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 425
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406746811,
     "user_tz": -120,
     "elapsed": 225118,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "14cfc94a-0ef0-4288-ae79-2b5909dd01e3"
   },
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "# Test SVC.\n",
    "tuned_parameters = [#{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],\n",
    "                    #  'C': [1, 10, 100, 1000]},\n",
    "                      {'kernel': ['linear'], 'C': [1]}]\n",
    "model = SVC()\n",
    "y_pred_svc, best_params_svc = dh.testSKLearnModel(model, tuned_parameters, train_features, valid_features, test_features, train_labels, val_labels, test_labels)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Model tested: <class 'sklearn.svm._classes.SVC'>\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'kernel': 'linear'}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.800 (+/-0.000) for {'C': 1, 'kernel': 'linear'}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set.\n",
      "The scores are computed on the full evaluation set.\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.851     0.754     0.800      1278\n",
      "           1      0.779     0.869     0.822      1280\n",
      "\n",
      "    accuracy                          0.811      2558\n",
      "   macro avg      0.815     0.811     0.811      2558\n",
      "weighted avg      0.815     0.811     0.811      2558\n",
      "\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "XKguSr8wrl9x",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "predictionLabels = [dictLabelsReverse[t] for t in y_pred_svc]\n",
    "dfResults = pd.DataFrame(predictionLabels, columns=[\"prediction\"])\n",
    "dfResults[\"gold\"] = dfTestDataset[[\"account.type\"]]\n",
    "dfResults[\"class_type\"] = dfTestDataset[[\"class_type\"]]\n",
    "file_name = resultsDir+\"/svc_bow.csv\"\n",
    "dfResults.to_csv(file_name, sep='\\t', encoding='utf-8')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mgsdzMkdXMLf",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 629
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595406750540,
     "user_tz": -120,
     "elapsed": 228832,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "27a3d600-decb-4fbf-8c43-0407ec9055a8"
   },
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Test logistic regression.\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "tuned_parameters = {'C': [1/64, 1/32, 1/16, 1/8, 1/4, 1/2, 1, 2, 4, 8, 16, 32, 64]}\n",
    "y_pred_log_reg, best_params_log_reg = dh.testSKLearnModel(model, tuned_parameters, train_features, valid_features, test_features, train_labels, val_labels, test_labels)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Model tested: <class 'sklearn.linear_model._logistic.LogisticRegression'>\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.699 (+/-0.000) for {'C': 0.015625}\n",
      "0.736 (+/-0.000) for {'C': 0.03125}\n",
      "0.763 (+/-0.000) for {'C': 0.0625}\n",
      "0.775 (+/-0.000) for {'C': 0.125}\n",
      "0.783 (+/-0.000) for {'C': 0.25}\n",
      "0.786 (+/-0.000) for {'C': 0.5}\n",
      "0.789 (+/-0.000) for {'C': 1}\n",
      "0.788 (+/-0.000) for {'C': 2}\n",
      "0.782 (+/-0.000) for {'C': 4}\n",
      "0.776 (+/-0.000) for {'C': 8}\n",
      "0.774 (+/-0.000) for {'C': 16}\n",
      "0.767 (+/-0.000) for {'C': 32}\n",
      "0.757 (+/-0.000) for {'C': 64}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set.\n",
      "The scores are computed on the full evaluation set.\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.841     0.749     0.792      1278\n",
      "           1      0.774     0.859     0.814      1280\n",
      "\n",
      "    accuracy                          0.804      2558\n",
      "   macro avg      0.807     0.804     0.803      2558\n",
      "weighted avg      0.807     0.804     0.803      2558\n",
      "\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "bkGgOU7oqiJb",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "predictionLabels = [dictLabelsReverse[t] for t in y_pred_log_reg]\n",
    "dfResults = pd.DataFrame(predictionLabels, columns=[\"prediction\"])\n",
    "dfResults[\"gold\"] = dfTestDataset[[\"account.type\"]]\n",
    "dfResults[\"class_type\"] = dfTestDataset[[\"class_type\"]]\n",
    "file_name = resultsDir+\"/logistic_regression_bow.csv\"\n",
    "dfResults.to_csv(file_name, sep='\\t', encoding='utf-8')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "xYKqumdxwCnC",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "X_test_all[\"label_log_reg\"] = [dictLabelsReverse[t] for t in y_pred_log_reg]"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "GCRSpAQ5PELE",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595407086573,
     "user_tz": -120,
     "elapsed": 564846,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "e957b4a7-8358-4112-91d2-4b360c66a55a"
   },
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "n_estimators = [100, 300, 500]\n",
    "max_depth = [5, 15, 30]\n",
    "min_samples_split = [2, 5, 10, 15]\n",
    "min_samples_leaf = [2, 5, 10] \n",
    "\n",
    "tuned_parameters = dict(n_estimators = n_estimators, max_depth = max_depth,  \n",
    "              min_samples_split = min_samples_split, \n",
    "            min_samples_leaf = min_samples_leaf)\n",
    "model = RandomForestClassifier(random_state = random_state)\n",
    "\n",
    "\n",
    "y_pred_rand_forest, best_params_rand_forest = dh.testSKLearnModel(model, tuned_parameters, train_features, valid_features, test_features, train_labels, val_labels, test_labels)"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Model tested: <class 'sklearn.ensemble._forest.RandomForestClassifier'>\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.681 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.677 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.694 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.681 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.675 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.693 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.680 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.677 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.693 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.692 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.680 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.677 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.691 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.680 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.677 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.691 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.680 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.677 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.691 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.680 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.676 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.691 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.692 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.692 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.692 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.679 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.692 (+/-0.000) for {'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.696 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.710 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.719 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.699 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.710 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.721 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.696 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.709 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.719 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.689 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.710 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.719 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.692 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.707 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.717 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.692 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.707 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.717 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.692 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.707 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.717 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.690 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.710 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.718 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.689 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.709 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.715 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.689 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.709 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.715 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.689 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.709 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.715 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.689 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.709 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.715 (+/-0.000) for {'max_depth': 15, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.753 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.768 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.768 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.755 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.768 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.767 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.754 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.768 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.765 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.764 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.766 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.765 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.755 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.766 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.760 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.755 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.766 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.760 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.755 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.766 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.760 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.755 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.763 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.760 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 5, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "0.720 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 100}\n",
      "0.742 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "0.741 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 2, 'n_estimators': 500}\n",
      "0.720 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 100}\n",
      "0.742 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "0.741 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 5, 'n_estimators': 500}\n",
      "0.720 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 100}\n",
      "0.742 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 300}\n",
      "0.741 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 10, 'n_estimators': 500}\n",
      "0.720 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 100}\n",
      "0.742 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 300}\n",
      "0.741 (+/-0.000) for {'max_depth': 30, 'min_samples_leaf': 10, 'min_samples_split': 15, 'n_estimators': 500}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set.\n",
      "The scores are computed on the full evaluation set.\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.759     0.798     0.778      1278\n",
      "           1      0.787     0.747     0.767      1280\n",
      "\n",
      "    accuracy                          0.772      2558\n",
      "   macro avg      0.773     0.772     0.772      2558\n",
      "weighted avg      0.773     0.772     0.772      2558\n",
      "\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "88nZLDI8rIx8",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "predictionLabels = [dictLabelsReverse[t] for t in y_pred_rand_forest]\n",
    "dfResults = pd.DataFrame(predictionLabels, columns=[\"prediction\"])\n",
    "dfResults[\"gold\"] = dfTestDataset[[\"account.type\"]]\n",
    "dfResults[\"class_type\"] = dfTestDataset[[\"class_type\"]]\n",
    "file_name = resultsDir+\"/random_forest_bow.csv\"\n",
    "dfResults.to_csv(file_name, sep='\\t', encoding='utf-8')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "w2vCK1FRrAOk",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "X_test_all[\"label_rand_forest\"] = [dictLabelsReverse[t] for t in y_pred_rand_forest]"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5z4JICTNSTbX",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def plotErrorRatio(X_test_all, prediction_column, min_sample_num=30):\n",
    "  X_verify = X_test_all[[\"screen_name\", \"text\", \"account.type\", prediction_column]]\n",
    "  X_count_accounts = X_test_all[[\"screen_name\", \"account.type\"]].groupby(\"screen_name\").count()\n",
    "  X_verify = X_verify[X_verify[\"account.type\"] != X_verify[prediction_column]]\n",
    "  X_verify = X_verify.groupby(\"screen_name\").count()[\"text\"].to_frame()\n",
    "  X_verify[\"total\"] = X_count_accounts[\"account.type\"]\n",
    "  X_verify.columns = [\"errors\", \"num_samples\"]\n",
    "  X_verify[\"error_ratio\"] = X_verify[\"errors\"] / X_verify[\"num_samples\"]\n",
    "  X_verify[X_verify[\"num_samples\"] >= min_sample_num][\"error_ratio\"].plot.bar(figsize=(15,5))\n",
    "  return X_verify"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "HkD_xLJ9zCRR",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595407087633,
     "user_tz": -120,
     "elapsed": 565878,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "cd509b8e-1081-4ece-be59-9bada48a7574"
   },
   "source": [
    "plotErrorRatio(X_test_all, \"label_log_reg\")"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>errors</th>\n",
       "      <th>num_samples</th>\n",
       "      <th>error_ratio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AINarendraModi</th>\n",
       "      <td>13</td>\n",
       "      <td>125</td>\n",
       "      <td>0.104000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeepDrumpf</th>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0.310345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DrilRnn</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GenePark</th>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>0.227273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GenePark_GPT2</th>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gpt2Wint</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JustinTrudeau</th>\n",
       "      <td>5</td>\n",
       "      <td>51</td>\n",
       "      <td>0.098039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Musk_from_Mars</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thorin</th>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>0.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UtilityLimb</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VBoterin</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VitalikButerin</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ahadsheriff</th>\n",
       "      <td>35</td>\n",
       "      <td>95</td>\n",
       "      <td>0.368421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ahadsheriffbot</th>\n",
       "      <td>12</td>\n",
       "      <td>95</td>\n",
       "      <td>0.126316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>awhalefact</th>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>botustrump</th>\n",
       "      <td>15</td>\n",
       "      <td>35</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calebgamman</th>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calebgamman2</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.153846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deep_potus</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deep_thorin</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dril</th>\n",
       "      <td>88</td>\n",
       "      <td>180</td>\n",
       "      <td>0.488889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dril_gpt2</th>\n",
       "      <td>35</td>\n",
       "      <td>179</td>\n",
       "      <td>0.195531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elonmusk</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2_trump</th>\n",
       "      <td>29</td>\n",
       "      <td>55</td>\n",
       "      <td>0.527273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imranye</th>\n",
       "      <td>34</td>\n",
       "      <td>129</td>\n",
       "      <td>0.263566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imranyebot</th>\n",
       "      <td>19</td>\n",
       "      <td>129</td>\n",
       "      <td>0.147287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jaden</th>\n",
       "      <td>19</td>\n",
       "      <td>51</td>\n",
       "      <td>0.372549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kevinhooke</th>\n",
       "      <td>10</td>\n",
       "      <td>241</td>\n",
       "      <td>0.041494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kevinhookebot</th>\n",
       "      <td>2</td>\n",
       "      <td>241</td>\n",
       "      <td>0.008299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>narendramodi</th>\n",
       "      <td>8</td>\n",
       "      <td>124</td>\n",
       "      <td>0.064516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ninjasexparty</th>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>0.739130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nsp_gpt2</th>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>0.173913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>realDonaldTrump</th>\n",
       "      <td>26</td>\n",
       "      <td>129</td>\n",
       "      <td>0.201550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sarcastic_trump</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>theJadenTrudeau</th>\n",
       "      <td>9</td>\n",
       "      <td>103</td>\n",
       "      <td>0.087379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whalefakes</th>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>0.057143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zawarbot</th>\n",
       "      <td>20</td>\n",
       "      <td>152</td>\n",
       "      <td>0.131579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zawvrk</th>\n",
       "      <td>54</td>\n",
       "      <td>152</td>\n",
       "      <td>0.355263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 errors  num_samples  error_ratio\n",
       "screen_name                                      \n",
       "AINarendraModi       13          125     0.104000\n",
       "DeepDrumpf            9           29     0.310345\n",
       "DrilRnn               1            1     1.000000\n",
       "GenePark              5           22     0.227273\n",
       "GenePark_GPT2         2           22     0.090909\n",
       "Gpt2Wint              1            2     0.500000\n",
       "JustinTrudeau         5           51     0.098039\n",
       "Musk_from_Mars        1            2     0.500000\n",
       "Thorin                3           10     0.300000\n",
       "UtilityLimb           3            4     0.750000\n",
       "VBoterin              1           13     0.076923\n",
       "VitalikButerin        1           13     0.076923\n",
       "ahadsheriff          35           95     0.368421\n",
       "ahadsheriffbot       12           95     0.126316\n",
       "awhalefact            5           35     0.142857\n",
       "botustrump           15           35     0.428571\n",
       "calebgamman           6           13     0.461538\n",
       "calebgamman2          2           13     0.153846\n",
       "deep_potus            2            3     0.666667\n",
       "deep_thorin           1           10     0.100000\n",
       "dril                 88          180     0.488889\n",
       "dril_gpt2            35          179     0.195531\n",
       "elonmusk              2            6     0.333333\n",
       "gpt2_trump           29           55     0.527273\n",
       "imranye              34          129     0.263566\n",
       "imranyebot           19          129     0.147287\n",
       "jaden                19           51     0.372549\n",
       "kevinhooke           10          241     0.041494\n",
       "kevinhookebot         2          241     0.008299\n",
       "narendramodi          8          124     0.064516\n",
       "ninjasexparty        17           23     0.739130\n",
       "nsp_gpt2              4           23     0.173913\n",
       "realDonaldTrump      26          129     0.201550\n",
       "sarcastic_trump       1            5     0.200000\n",
       "theJadenTrudeau       9          103     0.087379\n",
       "whalefakes            2           35     0.057143\n",
       "zawarbot             20          152     0.131579\n",
       "zawvrk               54          152     0.355263"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 30
    },
    {
     "output_type": "display_data",
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2oAAAGICAYAAAAwOBOrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5itZXnv8e+PJjbURGJFQGIJUWyAvZejwRINRBQbR4NGsR5NsBv0qGg0sSt2o1FRo6KgqFjAgnRBUY6IqFgiGgs2FLnPH8+79l4ze82eAWfP+yz293Nd+5p537Vmzb1nZq313k+571QVkiRJkqR+bDF2AJIkSZKkhUzUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOrPVWN/4qle9au20005jfXtJkiRJGtVJJ530k6raftZtoyVqO+20EyeeeOJY316SJEmSRpXkO0vd5tJHSZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6s9XYAUiSVt9OBx2x6o95zov3WvXHlCRJszmjJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdWVGiluSeSc5MclaSg2bc/ogk5yU5dfj3qNUPVZIkSZI2D1std4ckWwKvAe4OnAuckOTwqjpj0V3fW1UHboIYJUmSJGmzspIZtT2Bs6rq7Kr6PfAe4H6bNixJkiRJ2nytJFG7FvC9qeNzh3OL/V2S05K8P8kOsx4oyQFJTkxy4nnnnXcJwpUkSZKkS7/VKibyEWCnqtoN+CTw9ll3qqpDq2r3qtp9++23X6VvLUmSJEmXLitJ1L4PTM+QXXs4t05V/bSqLhgO3wTcYnXCkyRJkqTNz0oStROA6yXZOck2wL7A4dN3SHKNqcP7Al9fvRAlSZIkafOybNXHqrowyYHAUcCWwFuq6mtJDgZOrKrDgSckuS9wIfA/wCM2YcySJEmSdKm2bKIGUFVHAkcuOvecqc+fDjx9dUOTtLnZ6aAjVv0xz3nxXqv+mJIkSZvaahUTkSRJkiStEhM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ3ZauwAJG16Ox10xKo/5jkv3mvVH1OSJEmNM2qSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqzIoStST3THJmkrOSHLSR+/1dkkqy++qFKEmSJEmbl2UTtSRbAq8B7gXsCjwoya4z7ndF4InAl1c7SEmSJEnanKxkRm1P4KyqOruqfg+8B7jfjPs9HzgE+N0qxidJkiRJm52VJGrXAr43dXzucG6dJDcHdqiqIzb2QEkOSHJikhPPO++8ix2sJEmSJG0O/uRiIkm2AF4O/J/l7ltVh1bV7lW1+/bbb/+nfmtJkiRJulRaSaL2fWCHqeNrD+cmrgjcCPhsknOAWwGHW1BEkiRJki6ZlSRqJwDXS7Jzkm2AfYHDJzdW1S+q6qpVtVNV7QQcB9y3qk7cJBFLkiRJ0qXcsolaVV0IHAgcBXwdOKyqvpbk4CT33dQBSpIkSdLmZquV3KmqjgSOXHTuOUvc905/eliSJEmStPn6k4uJSJIkSZJWl4maJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1JkVNbyWJEn92+mgI1b9Mc958V6r/piSpOU5oyZJkiRJnXFGbTPjaKskSZLUP2fUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmPVR0mSJEmbrV6rojujJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHVmRYlaknsmOTPJWUkOmnH7Y5KcnuTUJJ9PsuvqhypJkiRJm4dlE7UkWwKvAe4F7Ao8aEYi9p9VdeOquinwEuDlqx6pJEmSJG0mVjKjtidwVlWdXVW/B94D3G/6DlX1y6nDywO1eiFKkiRJ0uZlqxXc51rA96aOzwVuufhOSR4HPAXYBrjLrAdKcgBwAMB1rnOdixurJEmSJG0WVq2YSFW9pqp2Af4ZeNYS9zm0qnavqt2333771frWkiRJknSpspJE7fvADlPH1x7OLeU9wN/+KUFJkiRJ0uZsJYnaCcD1kuycZBtgX+Dw6Tskud7U4V7AN1cvREmSJEnavCy7R62qLkxyIHAUsCXwlqr6WpKDgROr6nDgwCR3A/4A/Ax4+KYMWpIkSZIuzVZSTISqOhI4ctG550x9/sRVjkuSJEmSNlurVkxEkiRJkrQ6TNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSerMVmMHIEnafO100BGr/pjnvHivVX9MSZLWmjNqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1ZkWJWpJ7JjkzyVlJDppx+1OSnJHktCRHJ9lx9UOVJEmSpM3Dsolaki2B1wD3AnYFHpRk10V3OwXYvap2A94PvGS1A5UkSZKkzcVWK7jPnsBZVXU2QJL3APcDzpjcoao+M3X/44CHrGaQkiRJml87HXTEqj/mOS/ea9UfU+rJSpY+Xgv43tTxucO5pTwS+NisG5IckOTEJCeed955K49SkiRJkjYjq1pMJMlDgN2Bl866vaoOrardq2r37bfffjW/tSRJkiRdaqxk6eP3gR2mjq89nFsgyd2AZwJ3rKoLVic8SeqLy3ckSdJaWMmM2gnA9ZLsnGQbYF/g8Ok7JLkZ8AbgvlX149UPU5IkSZI2H8smalV1IXAgcBTwdeCwqvpakoOT3He420uBKwDvS3JqksOXeDhJkiRJ0jJWsvSRqjoSOHLRuedMfX63VY5LkiRJkjZbq1pMRJIkSZL0pzNRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpM1uNHYAkSb3b6aAjVv0xz3nxXqv+mJKkSw9n1CRJkiSpM86oSX8CR9klSZK0KTijJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BmrPkqSJEladVbH/tM4oyZJkiRJnXFGTV1yBEaSJEmbM2fUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzK0rUktwzyZlJzkpy0Izb75Dk5CQXJtl79cOUJEmSpM3HVsvdIcmWwGuAuwPnAickObyqzpi623eBRwBP3RRB7nTQEav6eOe8eK9VfTxJkiRJWk3LJmrAnsBZVXU2QJL3APcD1iVqVXXOcNtFmyBGSZIkSdqsrGTp47WA700dnzucu9iSHJDkxCQnnnfeeZfkISRJkiTpUm8lM2qrpqoOBQ4F2H333Wstv/emttrLM8ElmpIkSdLmaiUzat8Hdpg6vvZwTpIkSZK0CawkUTsBuF6SnZNsA+wLHL5pw5IkSZKkzdeyiVpVXQgcCBwFfB04rKq+luTgJPcFSLJHknOBfYA3JPnapgxakiRJki7NVrRHraqOBI5cdO45U5+fQFsSKUmSJEn6E62o4bUkSZIkae2sadVHSZIkqVdW8VZPnFGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ7YaOwBJkiRdMjsddMSqP+Y5L95r1R9T0sXnjJokSZIkdcZETZIkSZI649JHSZIkaY645HXz4IyaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1Bn7qEmSpDVlDyhJWp4zapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJntho7AEmSpB7tdNARq/6Y57x4r1V/TEmXTiuaUUtyzyRnJjkryUEzbr9MkvcOt385yU6rHagkSZIkbS6WTdSSbAm8BrgXsCvwoCS7LrrbI4GfVdVfAv8GHLLagUqSJEnS5mIlM2p7AmdV1dlV9XvgPcD9Ft3nfsDbh8/fD9w1SVYvTEmSJEnafKSqNn6HZG/gnlX1qOH4ocAtq+rAqft8dbjPucPxt4b7/GTRYx0AHDAc3gA4c7X+I4OrAj9Z9l7jM87VNQ9xzkOMYJyrzThXl3GunnmIEYxztRnn6pqHOOchRti849yxqrafdcOaFhOpqkOBQzfV4yc5sap231SPv1qMc3XNQ5zzECMY52ozztVlnKtnHmIE41xtxrm65iHOeYgRjHMpK1n6+H1gh6njaw/nZt4nyVbAlYCfrkaAkiRJkrS5WUmidgJwvSQ7J9kG2Bc4fNF9DgcePny+N/DpWm5NpSRJkiRppmWXPlbVhUkOBI4CtgTeUlVfS3IwcGJVHQ68GfiPJGcB/0NL5sawyZZVrjLjXF3zEOc8xAjGudqMc3UZ5+qZhxjBOFebca6ueYhzHmIE45xp2WIikiRJkqS1taKG15IkSZKktWOiJkmSJEmdMVGTJEmSpM6YqIkkO48dgyTNsySXWck5SZJWai6LiSS5S1V9OskDZt1eVf+11jEtJclngA1+yFV1lxHCmSnJSVV1iyRHV9Vdx45nObPi7DH2JIdU1T8vd64HSW4O3I72t/qFqjp55JDWWep5PtHT8x0gydZV9YdF565aVT8ZK6alDD/bye/981X1wZFD2kCSAPsB162qg5NcB7h6VR0/cmgLJDm5qm6+3LmxDL/rQ4C/ADL8q6rabtTAZkhyfeB1wNWq6kZJdgPuW1UvGDm0BZLsU1XvW+7cWJI8pKremeQps26vqpevdUwbk+TPgecBt2V4TQIOrqpu+vImedis81X1jrWOZWOS3BY4tap+neQhwM2BV1TVd0YObYEkT6yqVyx3bmxJDq6q50wdbwm8o6r229Tfe9ny/J26I/Bp4D4zbiugpwu3p059vi3wd8CFI8WylC2SPAO4/qwX9F5ezJNsC1wOuGqSq9AuNAC2A641WmBLuzuwOCm714xzo0ryHGAf1j9v3prkfR1dFM16nk9083xPcmfgP4Btk5wMHFBV5ww3f4L2RtmNJK8F/hJ493Dq0UnuVlWPGzGsWV4LXATcBTgYOB/4ALDHmEFNJLk67fXnskluxsLXpcuNFtiGXgLcp6q+PnYgK/BG4GnAGwCq6rQk/wn08po08XRgcVI269xYLj98vOKoUazce4BjaNdJ0AZo3gvcbbSINjT9urMtcFfgZKCrRI020HGTJDcB/g/wJlqMdxw1qg09HFiclD1ixrmx7ZDk6VX1omGlxGHAKWvxjedyRm3eJTm+qvYcO46JJDcA/hZ4EvD6xbdX1b+seVAzJHkiLcZrAj+YuumXwBur6tWjBLZIkn8EHgvsApw1ddMVgS+uxQjMxZHkTOAmVfW74fiytJG4G4wb2XxJcgLwiKHP5N7Ai4CHVtVxSU6pqpuNHOICSb4B/FUNbwJJtgC+VlV/NW5kC01mpaZ/hkm+UlU3GTs2gCQPp11Y7A6cOHXT+cDbepnxTfKFqrrt2HGsRJITqmqPRb/zU6vqpmPHBpDkXsDfAH9PSyQmtgN27en9fZ4k+WpV3WjRudOr6sZjxbScJFcG3lNV9xw7lmlTr5vPAb5fVW/ubIb/QcCDaSs6jp266YrARR2ukArwLuB04M7AkVX172vxvedyRm2pafyJXmaAAJL82dThFsAtgCuNFM5S7llVhyS5TFUdPHYwSxmmwl+R5PFV9aqx49mI/wQ+RrtQP2jq/PlV9T/jhLRRP6CNDP5uOL4M8P3xwllojpbvbFNVXwOoqvcn+TrwX0n+mRnLnztwFnAdYLIUZgcWDiz04g/DMpNJQrk9bYatC1X1duDtSf6uqj4wdjwbcWKS9wIfAi6YnOwlkVzkJ0l2Yf3vfG/gh+OGtMAPaEn5fYGTps6fDzx5lIhmSPLKjd1eVU9Yq1hW6BNJ9qXNVgDsDRw1Yjwr8Wugx33+5yd5OvBQ4PbDQNzWI8c07Yu05/RVgZdNnT8fOG2UiGYYtoVMvII2y/8F4JgkN1+LbSJzOaOW5LnDpzegTUMfPhzfBzi+qh4ySmAzJPk27c0mtCWP36atuf78qIFNmYxU9jTasjFJtgEeA9xhOPVZ4A2L9wWNLcmtaDMU5w/H29FmML48bmQLJfkQ7Xn0Sdrf6t2B44FzYfw38ySPrqo3TD3vF+hoxvdE4N5V9aOpc9cGPgrsUlVdLT9K8jna732y12sP2sXnLwCq6r4jhbZAkv2AB9KWjr6ddvH2rF72AU0MI+vPYf3r0udor/W/GC+q9ZK8dcbpqqr/vebBLCPJdYFDgdsAP6O9b+7X4f6arWnv7dcfTp3Z0/vQMNsLbc/Xrqyf/dsHOKOqHjNKYEtIcj5tueZkIGYLWiIEneynTPIR1g+8bUH7uR5WVQct/VVrb1iS/WDghKo6dtjbe6fe9tIBJLka65eUHl9VPx4znmlDnYml1FrUm5jLRG0iyTHAXlMXwlcEjqiqO2z8KzUtybtpy3auCXxr+ibaH+JuowS2hCRvoo0MvX049VDgj1X1qPGi2lCSU4CbL1padmJvyfDUm/lMw4zBqIYZlSdU1b+NHctSktwNOK+qvrLo/JWAA6vq/44T2WxJNrpXoao+t1axLCfJDWl7QQIc3eM+qyQfAL7Kwtelm1TVRovhaENJblFVJyW5PLBFVZ2f5N5V9dGxY5s2PIfeAZxD+9vcAXh4VR0zZlyLJTkOuF1VXTgcbw0cW1W3Gjey+bPodfNC4DtVde5Y8WxMkh2B61XVp5JcDthycr3ciyT7AP9KG3APcHvgaVX1/jHjmjZcu+1TVe9d9s6b4vvPeaJ2JrBbVV0wHF8GOK23vTVJbkQbddl2cq63UY1h9OUo2lKOBTocxdxgf0pPe1YmZu2pSHJab4nvvOhtb+dS5qWK1cQw07tuGXxvy3OTvAx4c1WdMXYsG7PE872nfVVzUUkR2v4a4GFV9dXheF/gyVV1y3EjWyjJScCDq+rM4fj6wLur6hbjRrbQcK1068lzO60Y13G9XSsBDH+XO7HwNanH5bldS/IPwAHAn1XVLkmuB7y+w71fXwHuPplFG5a2f6rD67kTq2r3Mb73XO5Rm/IO4PgkH6Rl4vcD3jZqRIsMy7XuREvUjqRV/fs8nVUIGpZrdfXE2Ig/Jtmlqr4F65bJ/HHkmGY5O8kTaBdH0AqMnD1iPDMluTfwfGBH2mtCr2W7v5Dk1bTlO5PlMKzFGvGLaS6qWCU5gFZF8Xe0pUahLem57phxzfB14I1JtgLeSrsQ7mI54SK/TXK7ybL2tPLYvx05pmnzUkkR2vLW9yd5MG2E/WHAPcYNaaatJ0kaQFX9v2G2qjcvBk4ZlnGFtjz3eaNGNEOStwC7AV9j/fLHbir7wrotDa8C/grYBtgS+HWH75ePA/YEvgxQVd9M8hfjhjTTFouWOv6UPns8fyrJU9nw+mOTD2zO9YwarNvod3vak/nYqlqTcpkrleR0WgJ0SlXdZFiL+86quvvIoa2T5LCq+vsh1uk/iF6XPt6VdsF2Ni3GHYH9q2pja4nX3PCi+EpaWfECjgae1NP6a4AkZwEPAE6vjl8QptaKT2Kc/H120ZMw81fF6pu0Ufbu+rvNkladdn/gQbTN3G/s6Tmf5Ka0ZY9Xov1t/g9tGVwXG+PTeSXFxYbZqQ8B3wXuX1U9Jb3AusTiIuCdw6n9aMvLetz3d3XglrTXz+On99L2IskZVbXr2HFszLAXeV9aC4bdaYMI16+qp48a2CJJvlxVt5w834eBrpM7vJ57KS05n7SJeSBtZVxvbYy+PeN0VdUmH9ic9xk1aDMpF9FefLqpBDblt1V1UZILhyVGP6atY+/JE4eP9x41ihWqqqOHafzJso0zJ8tfezIkZPuOHccKfA/4aq9JWtZXe/wo6wvzTPQU81xUsZryLeA3YwexEsMexRsO/34CfAV4ylBopovnWFWdSutbtN1w/MuRQ1qs90qKzBgs/DPajMWXk9DbRSbwj7SZi0nBpWNpff96tCdtUBvaz/gjI8aylC8l2bX3Zc5VdVaSLavqj7S+o6fQ+uf15HNp/XEvm+TutBU93f3Oq+ppSR5AG+AEOLSqPjhmTLNU1WiVPec6UUvrq/UPtOanAd6Z5NDqq3T7iWnVwN5IK+P7K+BL44a0UFX9cLgQeltV3XnseJaT1vj6sbQndgHHJnl9DX3AejHE+Ujgr1m4P7G30dZ/Ao5MqwI4Xba7l7L3k2qJkyqvH6Y93+/D+oqFoxv2cn4HuPUwer0n7e/zzMkm/s48Hfhiki+z8PfeVcnuJP9G+10fDbywqia/80OGvTddSPLnwHMZXpeSfJ5W9fGn40a2zuNolRRvmOT7tEqK3VRIHszFYOFEVV0wLMc+mjZQfGZV/X7ksDaQ5MW01853DaeekOTWVfWMEcOa5R20ZO1HtNekHlf1/Cat8vSpSV5CG+zocaneQbTrj9OBR9P6fr1x3JCW9EXWT7qcMHIsMyU5jTbr996qWtMtLHO99HH4wd26qn49HF8e+FJnT+p1kuwEbNfLUpjFkhwNPKDT/R/rJDmMNksxWW7yYODKVbXPeFFtKMn7gG/Q4juYtizm61X1xI1+4RpL8gnaAMLpTM1KVydl7ycyJ1VekzySdsH+adqFxh1pF+xvGTWwRZIcT9svu/j3PnqVz2lJ9qeVv/71jNuu1MvrVZJPAsewcBncnarqbuNFtaFMVVIcO5aNSXIT1s8AHVuLqqn2IMlewOtps9Oh9dN6dFV9bNTAFhmulW5aVRcNx1vStmN0da00LMN/Chu+JnVT0CytkuJ/0/anPZm21Pm1VdVVD8okB1fVc6aOtwTeUVX7jRjWBpI8itbWpPf3yx1pyzIfSPvbfC/tfem7m/x7z3midjqwx2QmZZjBOKE66mKfJLQ37OtW1cFpvSyuPjUq3I0kHwZuRuunNb1ZsrcR9g3Wsfe4tn1qbfhpVbVbOi2JnOSrVXWjseNYTuanyuuZwG0mMynDTMsXO4xz3V6l3iW5FuuL3QBQ/ZVA3+B5lOT0sd+PskSj+ImOZs7XmVotMykicX/akqieVsuQ5Bu03olnDce70AaPbjhuZAsNidqdan3Vxz8DPtthovalqrr12HEsJ8llgetMF5LpTVrfxP9XVS8aZgAPA06tqueNG9lC8/J+OW3YevNsWm/HLTf195vrpY+0ghJfTqv6CPC3wJtHjGeW19Ky77vQZlXOpy3V3GNjXzSS/6Kj6kobcXKSW1XVcQBJbklr1NubSePTn6e1aPgR0GPVpSOT3KOqPjF2IMuYrvIK7fn+tvHCWdJPac/zifOHc735WFrlx4+wcOljb+X5X0zb63kG66u7Fm32qiefSCsjf9hwvDet5cnYFi8dPnw47mrp8CKPBG45tVrmENqWga4SNeD8RTMpZ7Pwud+LF7Fh1ceuGjQPTkmrRLr4Namb65Ik96H1/doG2HkoInRwVW3Q2mhk/xt4V5KnA3cGPlZ99iGdl/fLxbNqf6RtG9n033eeZ9RgXdXHySbEHqs+nlxVN8/CSlvd9fya6HmkaGqj+da0i47vDsc7At/ocEbtUbSk/Ma0hOIKwLOr6g1jxrVYkvOBy9PeGP9Av+X5p6u8AhzT2/MdIMk7aL/zD9P+Pu9HKyZyGvQzgzFmFauLY/FMam+G58+kyM3lWV/cakvgV708j+Zl6TD0v1pmKH4AcHfa+89htN/5PsB3q+qxY8W2lCTXYP0Aca9VH98643T1tK87rXfeXWgzkpNrutFnzieG98iJrWntOL7AMIlRnbSzmZrpvykz3i+r6hEjhTbTsJd7a1q1zzXdpzaXM2rDtP3EOcO/dbd1NhpAXa4AACAASURBVCL8h2Ft8KTS1vb0WZ1yHkaK7kef/dIWyPrmxl+vqp/RRv67uvidVlVXXP5efRjeZLp4o9mIbw3/Jj48fOzq5zxmFauL6WzaG2SXidocPX+uBkwXuvj9cK5H06tlJj1Se1otc5+pz/+btq8G4DymCkd1Zvvh41bAbdKqaHYzUwVQVfuPHcMK/KGqftF2tazT04zHyxYd/4zWx/dltDi7aGfD+vfDpd4ve/PQqvp/Y3zjuZxRS3IRcC4wqaS2oFx3TyPCSfajTZPenNZjZ2/gWVX1vlEDm2GJkaJu9i8lOamqbpHk6OqsJ9W0DL2JJrOpY8eznCQzR9R72wOk1ZXkYbPOV9U71jqWjUnyAVovyqPpuzrlZD/yzlX1/CQ7ANfoZT9ykmcCfw9MLx0+rKpeOF5US5taLVPA53ucPZ8XWaKRdE8zVbBuRm2Di9Ke4kzyZtpr0UHA39FaM2xdVY8ZNbA5NExiHFJVTx07luUk+RZwHK0Fx7FV9bW1+t5zOaNGayJ8Z9p07rtpL+JdZpxV9a4hAborLaH826r6+shhLWXWSFFPs39bpPUFuf6sDfK9LCkDvp7WTPiawybuiR5LDQM8berzbWll5SdJuy6mJB9hw4uNX9D2Ub6h+mkjMb1Pdlvaa9TJtL2APTmc9fuqeja9H/n5tEqqr6GT/chV9X+TfJz1WwX27zz5+SPtedRrj9RJU+7XAVerqhsl2Q24b1W9YOTQFrtVb1sDlvDRqc+3pRWR+cFIsSzl8cAzaYNG76btQ33+qBEtYahKurg90MHjRbRQVf0xyW3HjmOFdqU1jL898NIkN6At0bz/pv7Gc5moVdWThtHLOwEPBV6VVmL8dVU1a9/Fmlu0PPPHrO+63uPyzImvJXkwsOVQ1eYJtP4WvdiXNgq8FZ0tI5tWVQ9K66N1FNDLstElVdX0Mh6GmYB/HymcS4OzacuMJs/5B9I2SF+f1k/xoSPFtUBVPX76OK3f43tGCmdJ1Vm7gI245WQ/MkBV/WyottaNqjopyfcYLtySXKfWoLz0xZX56JEK7fn8NNo+IKrqtKEYRm+J2rw0kv7A9HGSd9NaiHSjqn5DS9SeOXYsG5Pk9cDlaJMab6Kt5upidn+RU5McTtv7NV1tvKtlubSBoz+wvt/bj4d/m9xcJmrQpiWAzwxvivvSRjS+SXvh7MFJrN9gfh3aOuEAV6YVwehxf0jvI0X3rKpDklymp1GhWYaN2jfpuTjLRpwL/NXYQcyx21TV9CzKR5KcUFV7JFmz5RKXwK/p8HVpGDR6EW1Ec3pkuJsl7oOu9yMnuS9tn8o1aRcY16H1efzrMeNawrxUfbxcVR2/aBVKj83t56GR9CzXo5NKyUuslFino738E7ep1hbotKr6lyQvA7rq7zfYllblcXoFT9FfBfJf0vr7vRx446SdwFqYy0QtrWHn/Wgj1dvTfqG36GlkcLJRP8kbgQ9W1ZHD8b1os0LdmYORov2BV9B+fl0najAXxVkASPIq1r8BbUGrwtR7wY6eXWF6piKtd+IVhtt+v/SXra1FFx5b0BKhw5b+itG8ldZA/N9oo8P70+LtzStp+7/+Isn/ZdiPPG5ICzwfuBXwqWr9He8MPGTkmJYSFhaO+iML96L34idpvdMmyfnewA/HDWmmN9Nm8hc0ku7NVAXViR8B/zxSOIv96/DxAcDVWd/Y/kG0gjK9mSyx/02Sa9KSoWuMGM9Mc1JABtrv+XbAY4FHJfkirfL00Zv6G89rMZFf02bP3jN8XPCf6GnKdFbZ1p5KuU4b1ts/FdiJhY1lu9irNCyD2J02IjxdJajL0cHey/hOJHn41OGFwDlV9YWx4pl3Sf4GeD3tbzS0WarHAp8F/qGqulhWmuSOU4cXAt+pqnPHimcpU0WE1j13JufGjm2xJDdk/X7ko3vaj5zkxKraPclXgJtV1UXptFXMsAf54SwsfPK2Xp47E0muCxwK3Ia2aubbtCa43xk1sEUyB42kh+0sO/Q04D7L5Hm03LmxJXk2bQb6rrS9skWbCXrOqIEtktZ645FsuJeumwIy04bX+HsBTwL+oqouu6m/51zOqNHWshatl9bi7uW9TZn+IMmzWD/6sh/9bY6deB/tAvNNdFgGf972ftF/Gd9J1aV7VNV+Y8dyaVFVRw7L9W44nDpzqoDIvye5e1V9cqTwgHW/9+dV1Z3HjGOFLkiyBfDNJAcC32f9DGU3hqVFb66q14wdyxJ+nuQKtKpl70ryY6b2hPSkql6e5HPApNBAr4VPrlJVdxtW+WxRVecnuTfQVaLGHDSSrqpKcgStp1bPLp/kujX00UqyM61/YheS7DNUFX9nVf0c+ECSjwLbVtUvRg5vlv+gLcH+X7SVUvsB3QxwTUxVH/4WreXSw4Avr8n3nscZtXkyFBV5LjApgX4M8C89FhPpdZR6Xs1LGd8knwfuUlXdLMu7NEsnbRuSHA08oNM373WS7EF7474ybfnedsBLq+q4UQNbJK3B/f60AdC3Au/u6Web5HK05VChLXncDnhXj+9FsG4w4WosXN3R1WxLkpOBh1XVV4fjfYEnV9Utx41socxBI2mAJG8HXl1VJ4wdy1KS3JM2i3o27bm0I/Doqjpq1MAGk/eXXt5nlpPklGEp9mnDnrqtaeXvbzV2bNOS7A6cUlVrPokx94la7+VH58FUhcon0DaZf5CFo25dvJEnOayq/j7J6Sycmep16ePlaPv97kGL8Sjg+R2VZwcgyTtoxUMOZ2HVpV7aHVyqTN6YOojjw8DNgE+y8PfeTX+yzFGfnYm0ss370/Y0fIG23OgzI8bz+aq63aL9P5Np/ouA/6Elvq8dJcAZkjyeNsD536zfn9bja/x1gfcDD6aV7X4YcO+eEvR5kOTAqnp1km8Af0mbkfw1/f7eL8P6FRPfqKoLNnb/tZTkk7Tn+R602fMFOtwjf3xV7ZnkGNoWgR8Bx3dYLIokN2LDolabvJ3NXCdqS5UfrapHjhrYlCSfYXYDxy72fQEk+TbrK1QuVr08YZJco6p+mGTHWbf3ti9gXiR57ozT5YDHptHLSOeivYkTtRZvPBdHkuN6G11dypBY3puWqO1AK85yO+DXVbXvmLEtJcmfA1+sqsXbCEaT5Cxa1cc1q6x2SQ17uz9Eq+Z8/6r67cghbaD3fUBTs0Ddv7cnedis8728bqa1BLk5bUnhoxbfXlWfW/OgNmJYifABWkP2t9KWtT+7qt4wamCLDNdJd6IlakfS9ql9vqr23tTfe173qE3MQ/nR6ZHgbWlL4Loq31tDhcreDUnalrRN5d3urcn8lfE9Y1jTvk6SfcYKRmvmylX1iukTaf2renNK5qDPTpJ/A+5DW+78wqqa9Cw6JEm37Tmq6qdJ7jR2HIt8j9YkvkszVnX8GbAl8OUk9DYDxJzsA+opIduI6dYr29KKdZxMa4EwumELw3FJblNV5yW5XLWK3l2qqjcNn34O6GJSYAl70/aonVJV+ye5GutrT2xS856odV9+tKpOWnTqC0l6bDo4uTj/+LAh+lm0UZnn97SJu1on+4uSXKnj5SX/uvxduvJ02kXwcue0Os4ZO4DBw2ntLqY9Ysa5sc1Ln53TgGfV0PtrkT3XOpiLo6q6KCk/VHuEtv/ns0Nxiell+L0sx7732AFcTH9ZVfskuV9VvX0oLLLBsrgR7ZbklzPOT5Y+brfWAS2lqh4/fZzkyrQK5L35y6EgzxWA6yS5CW0v3WNHjmuBJN8CjqP9PR5bVb32Gv1dtUq5FybZjvW9KDe5eU/UPjI8SV5KG9Eo+ml4DSzY/wWt988tgCuNFM5ynl1V70tyO+ButJ/r64GuNkYDvwJOH9Zid7e3prelBUtJ6+n3N8C1krxy6qbt6GzWdx4kecDGbp/MAFXVRu+3qSV5EG1Pzc7DTNXEFWn7lbpSc9Jnp6remuRawwXRdAGMYzoeVOrNFYeP3x3+bTP868r0zM/w+779cHhsVX1lnKg26g/Dx58P+2x+RCeNpAen97Bv9xL6Na0FS2/+nTaDejhAVX0lyR02/iWj2JV2jXl74KXDHt/Tqur+44a1gROGfOONwEm069A1aWM0t4laWrnmo+eg/OhJrN//dSGtz0o3e+gWmVSz2Qs4tKqOSPKCMQNawn/R32j6BtJKtL+IDTef9jK9/wPgRFqrg+mZ3/OBJ48S0Xy7z0Zu62kG6Iu0prxXBV42df582qxQV4by149nw/6OXS0hTvJiYF/gDNa/lhat0q9WoKr+ZewYLo5hqfA/sP65/c4kh1bVq0YMa5ZDk1yF1oD9cIZ9QOOGNJ8WbW3Ygvb+fth4ES2tqr6Xhe2Bumu7RIvpD8PHi2gzVT8eNaLZbkjbbvX6JB+nTbg8bi2+8bwXE+mietpShmTy1jUnzYOHZPf7wN1pyx5/SyvO0mND1MsC16mqbvd+DGXvnwtM9q7sT+u101vDya2r6g/D51ehNR3t7oJdayudNMlNa9D8ZuB02hs50N/M9bAPbbeeKsDNq6FAx1PZMDnvpggXQJLTaO/xvx6OLw98qac9asN1yN5V1WUyAZDkGVX1wrHjWIkkd5w6vBD4TlWdO1Y8S0nyfuDlwKtpM1ZPBHbvrahRkt/QXttfDnyq1wJCSc6m7Z399GRAaa0Kg817ovavwJeA/6pO/yO9J5PThnLy96QtQ/hmkmsAN66qT4wc2gJJ7kPbB7ZNVe2c5KbAwR2OsJ9UVbdIcnpV3Xj63NixTUvyWdqs2la0mbUf06rAOat2MSR5SFW9c2qfzQId7a9ZkV5eu5J8uTrrSzVLko8B+1TVr8aOZd4Nyfnraa9H62YBZuz5HtVQVGSPGlquDNUVT5i83vciyYlVtfvYcSwlyavYeAGuLrY1DMXMPtVzMbOJJFel7Te+G21F1yeAJ/aWCCW5H60y7p7A72krPo6pqqNHDWyRtJ6JewKvpFX0fQjwmbVI1OZ26ePg0cBTgAuTTBp5drXxFDg6yd/RdzI5vY/us1PnLqAtjevN82hPmM8CVNWpaf1senPBMJr5zSQH0mYrrzByTLNcqap+OZTJfUdVPXcYKdbFc/nh4xU3eq/50cvr1SuG0sifYGFhiZPHC2mm3wCnpjUSn46zi4vMOXNhVb1u7CBW4K20So8fpF1/3I82+9ubTyV5KvBeFu7r7mVP6uQ647a0pYTvHY73oS0l7sKcFDMDoKp+Qqvu2bWq+jDw4SQ3pJW8fxLwT8BlRw1sQ6mqC4HHJnkE8HngKmvxjec6Uauqbi+Ikrywqp7BfCST0/vorgP8bPj8yrQN3b1tlP1DVf1i0drri5a684ieSOvz9wTg+bSqdbN6V41tq2H29O9pDbp1CVTVG4YR119W1b+NHc+lyI2Bh9KeP5PnebGwCmQPDh/+6U/3kSSPBT7IwqS3l8QCaLPkw4qE29H+JvfvqUrylAcOH6f31BSdlEOvqrcDJPlH4HbDBfGkV24X1SmT3KqqjqPzYmbzMjs5keQDtLL336Lt530Y8OVRg5rt9ZNPquptw2z6muxRm8tELclGpxo7GWm9J/CMnpPJiRr6qCV5I/DBqjpyOL4X8LdjxraEryV5MLDlULDjCbTp8q5U1QnDp7+i7U/r1cHAUbTmjScMs5PfHDmmuTSMuD6Iti9x3mX5u6yJfYDrVusP1K3JxaZWxWRA62lT57pJLBb5Iy22os8Bw7nplUqbodiO9dVnr8AazVqswGtpe/d7L2bW4yqojXkRrTdZj4VO1qlFDbiHZdhr0jB+LveoJfnM8Om2wO7AV2gXFbsBJ3a0Af5OLHGx09vIILT19ovX1s86N7ZhL90zgXvQfr5H0fq9/W6jX7jGhg3xTwN2pOMN8VpdaY2Pt2bDZUY9DCCtWJIbVdVXO4jjQ8ABVdVjJbB15qDKq1bZVNXHD9Dei+5Pq5jcRdXHlbYM6UWS/WlbGz5D+3neAXheD4Mga1U4YnM0tIxY/LrZRQPxHsxlojaR5L+A51bV6cPxjWhP6r3HjQySXEDbkzQrUase37yTHEVbZjDptr4fcIeq+l/jRTW/5mhD/FuZsVSiqtZktOjSZmogafIznSx37iJBT7IDrUfitYCPAS+dqvr5oarqahZ9WFq2G3ACC5fB9VY8aC6qvPYsyV2q6tNLJRgdJhZdV30cXtuh9Uy7DfDp4fjOtIJR3TXuTnJ11vdu/XJV/WjMeCaS/JyNtNro8PVoe+Cf2TAB6uJ9aGLYf3wnWpxH0vapfb6H6/hezOXSxyk3mCRpAFX11SR/NWZAU87ooWLaxfQg2oXGB4fjY4ZzXZmX0s3Mz4b4j059vi1tVPgHI8Uyt6aqPX6U9Xs+J3oaEXsLbQbgOFpPx88luc9QDWzHUSOb7bljB7BCl62qo5OkWkPk5yU5CTBRW7k70pKJWT0Je+pFOBEW9qb6I/0sGV7XLD7JJ4Bdq+qHw/E1gLeNGNoCM7azfG/4eM0k1+xkNcJ5LOw72bt30VZ17AU8hrac+LxRI5ptb9oetVOqav8kV2P9ZIGY/0TttCRvYuEMkNXqLqFhOeYTx45jBd5Hm6l6Ex02cJyqojkvG+I/MH2c5N20ika6eCb7UW8A7AF8mHbRdh/g+LGCmmH7qppsjH58kocAxyS5L30llEB//dI2Yl6qvHarqiZJ+aN637MymK76CG1Pd49VH3eYJGmD/6YVDuvFxhKgXgoHnT9Hr0UAf15Vb07yxCHuzyU5YdmvWnu/q6qLklyYZDtae6Ce/jZHN++J2v7AP7I+uTgG6GUG4xVjB3BxDVPl/wT8NR1PldP/TNV0FU2Yjw3x065HWyqji6HWN8E8Brh5VZ0/HD8POGLE0BbbOsm2kz2d1Xq//Yi21/PyG//StZPk81V1uyTnszCB7LFyLmxY5fXO9FnldR58O8nHaTMCn+61tc1Q9fFztLLy0G/Vx6OHrQ3vHo4fCHxqxHgWmIe+ZMA5YwdwMf1h+PjDJHvRVsn82UbuP5YTklwZeCPt2ulXwBfGDakvc71HbR7MU0GJYXnEe2nLCtdNlVfVP48a2GBqpuoJtFGXrmeq5sWMC+EfAU9fPNOmlUlyJrBbVV0wHF8GOK2qbjBuZE2SJwMnLx4dTnIz4CVVdfdxIptfQ1uGQ6rqqWPHcmkwFIy6N7AvrdLeR4H3VFV3M/3D7/5qLHx//+54Ec027Pu7/XB4TFV9cGP3H8s8FJZIchs23HrRW4z3ptUc2AF4Fa2a5r9UVVctRJJ8GvjXqjoyyU7AlYDHVdUBowbWkblO1JLcllYhaHES1M2MxbwUlABIclJV3SLJaZPN0ElOqKo9xo4NIMm32XDvz0R3BVqG4gKfo71YfmEyw6JLtyTPpPWkm14O9d6qetF4UW0oyW2r6gvLndPKJDmuqm41dhyXNkmuQluhsl9VbTl2PNOSPJ62h/K/Wb8/rXopJjJv5qGwRJL/AHYBTmX9NV1VZ/3J5kWSs2l7Ej89tSrFCptT5j1R+wbwZDZMgn46WlCLTJKfseNYicmFxrBE4pW0qfL3V9UuI4c2l5LsTBvBvD1wK9rs37FV9eRRA1skydFVddflzmnlhs3x06PX3S2HmvVm6BvkJZfkdbRKmu9jYVuG3gpgzIUkd6Qt0bsnrTfUe3ub5U9yFnDLnq45Zhlm0w6hLWkPnS4fTmsiPCkscZNJYYmeZvmTfJ1WmKXri+dhK8s/sOHMX1fVnJOcDOxJu+bcAXgI8Bnfh9ab9z1qv6iqj40dxDLmoqDE4AVJrgT8H9ZPlXeVVAAk2Qf4eFWdn+RZtKUxz+/tYriqvp3kd8Dvh393BnqpSkqSbWl7aq46jFpPZiq3o11w6hIaqpT1UKlsA0luTSvVvf1UpUpov/euZizmzLbAT1lY+KDHSoXdS3IOcApwGPC0Sfn7Dn0P+MXYQazAS4D7VNXXxw5kGb+dUVhih7GDWuSrwNWBHy53x5F9mLaa51N0WHRtSqrqQuCxSR5BK2TWS5PzLsx7ovaZJC+lvRFOJ0E9XSBNNpN3X1CiqiZl2n9BSyp69eyqel+S2wF3o/WEej3re690Icm3gJ8A/0mrBPb4qrpo3KgWeDTwJOCaLEwqfgm8epSItBa2oVUj3Ir1lSqh/d67WWI0byal0LUqdquqX44dxFKmBjjOBj6b5AgWXoO8fJTAlvbfc5CkAZw4o7DEl8YNaQNXBc5Icjwd93UELtdLfYFlTCoQU1VvG2ZVHzdiPN2Z96WPn5lxunos1DEP5miq/JSqulmSFwGnV9V/Ts6NHdu0JE8EbkcbEfwGbb/aMVX1rVEDWyTJ46vqVWPHobWVZMeq+s4wcl3uofzTDEudH8+Gr5+9XcB1byjC9TrgalV1oyS7AfetqheMHBqwbi/VkiZ7bXqR5BW0WaAPsTC56Ha2dygssV1VddVyaViSu4HeSvcneQGtqfmRY8eiP81cJ2o9S3KXqvr0sDZ8Az2+QCb5Im2qfPGev972BXyU1qPo7rRlj78Fjq+qm4wa2BKSXIHWSuKpwLU73BC/Da3K5x2GU58F3lBVf1jyizT3kuxO6wM1mVX7BfC/eyx0NA+GwlFvBk4H1s2c93YBNw+GkvdPo70O3Ww499WqutG4kc2nJG+dcbp6G4QFSHItNiwQd8x4EW0oyY7A9arqU0OF0i17GehaVMX5CrTE/MLhuLt9iVreXC99HDaavhC4ZlXdK8muwK2rqoeGk3cEPk1rdrtYr/sW5mWq/O9pG8z/tap+nuQaLFxa2oUkL6MVlLg88EXgObREuDevBbYePgI8lDaa/ajRItJaeAvw2Ko6FmBYSvxWwIp1l8zvquqVYwdxKXG5qjo+WVDg98Kl7jyWYebvqWw4i9rVqp55WZab5BBaAZkzmKqoSOuR24Uk/wAcQOtJtgttP/frgS6Kb1XVFQGSvJP2czt2Tpa9aglzPaOW5GO0C4tnDhWCtqJVC7rxyKGtk2Tnqvr2cud60PtU+VQftZl6K9CSZG9a48brAJeZnO9wdPAri2cjZ53Tpcus5cJWfbzkkjyY1iz+E/S7Z3ouDO/tBwLvq6qbD6+lj6yqe40c2gLz0n6n96WkE1nUg7JHSU6lVSn88tRs7+k9XXcCJLkz66tO70Lbh35sVb1i1MB0sc31jBpw1ao6LMnTAarqwiS9Vbf5AG153rT3A92U7J+aKg/wjCQX0Lra91bC9yTWx3kd4GfD51cGvgvsPF5oM12FdtF2bVrPlVvRNkZ3NdoK/DHJLpO9c0muS99VorQ6PpfkDcC7ac+rB9IKI9wcTDAugRvTZqPvwvqlj0V/z/d58FzgDcANk3wf+Dbw3nFDmunCqnrd2EGswBsZlpICVNVpSf4T6CpRoxVn2ZqpgY4OXVBVv5/M9g4TBN3NeFTVZ5IcA+xBKw73GOBGtJ6EmiPznqj9OsmfMzxJktyKTkrlJrkh8NfAlRbtU9uOVsa5G5Opclg3a3U9OosRoKp2BkjyRuCDk5m/JPeiNRXuzRNoL5LHVdWdh7+JF44c0yxPo1VQPXs43om2p06XbpMZ0+cMHyfrzG6GCcYlsQ9w3ar6/diBXAq8nlYx+WxgC+DetAq1vRU9mpf2O3OxlBT4DXBqkqNZ+PPsqZn055I8A7hskrsDjwU+MnJMGxh+hpenDQ4fC+xRVT8eNypdEvOeqD0FOBzYJckXgO3pp7z0DWhvLldm4T6182mVFbuT5FHAE1k4A/RFOll7PeVWVbXuZ1hVH0vykjEDWsLvqup3SUhymar6RpIbjB3UDF+gjbTeFfg5cBT9lUTW6vso62eoGT7/BXBSVZ06WlTz66u013svhv50e9Mahz+YtnTrYcA9Ro1otnlpv/OTJLuwflB7b/rsA3b48K9nBwGPpBUNejRwJPCmUSOa7TTayq0b0V7Xf57kS1X123HD0sU1t3vUkmxJm7F4FS0pCnBmb5Xqkty6qubionfoXzGZAbrpZAaoqmZWrhxLkqNoI0TvHE7tB9yhqv7XeFFtKMkHaTNTT6LNTvwM2Lqq/mbUwBZJchith9a7hlMPBq5cVfuMF5U2tWHp0+60C6PQBpZOo1Vce39V9Tj40a0kn6UVYjmBvvsrzYVhX9WHaMva7+8F5iU3LGc/lNbo/me0paT7VdV3Rg1shqEK8fWHw+6u6eZNkisCj6AVvbl6VV1m41+h3sxtogaQ5Piq2nPsODZmmOl5Aa2E/Mdpb+RPrqp3bvQLR5DkhKraY9gse8uquiDJ16rqr8eObdqwPPO5rC8nfwzwLx0uN1ln6L1yJeDjvS2NSnJGVe263Dldugz7F/6mqn41HF8BOIJWUfUkf/8Xz7z0V+rZMFg4fVHyF7TZgAsAqqqLiqTz1H5nGNQ+pKqemuTywBa9lJJfLMmdgLcD59AGj3YAHt5DAa4Zf5sL9PK3OZHkQNps9C1oP89jacVEPj1mXLr45n3p4xeSvJq2yfjXk5OdbYK/R1X9U5L7054sD6AlFt0lasC5Sa5MG8X8ZJKfAd2NuA0J2RPHjuPi6Pxi7eQkt6qq4wCS3BI4ceSYtOn9BQs37f+BVhXut0NBIV0MnT/H58W9xw5gheam/U5V/XFovUFV/Xq5+4/sZbRrpjNh3azqu+mj+Nrkb/Nxw8f/GD4+hA6LidDqDLycNujW435ErdC8z6h9Zsbp6qmHyWRGKsmbaMuJPj4Ppc87nwHaHvgnWrGWdUVPevq9z4OpEcKtacuHvzsc7wh8wxmVS7ckzwbuD3x4OHUf2jLIlwGHVtV+Y8U2T5J8vqput6jRLPRXNVerLMmWVdV9hdwkr6P1+3ofCwe1u0koAZKctnhmata5MdnWRGttrmfUqurOY8ewAh9J8g3a0sd/HJKM340c07I6Hx1+F20W9d60krMPB84bNaL5NC+j19oEqur5Q7+q2w6nHlNVk5lUk7QVqqrJbMUVl7uvLnW+neTjQjW2dQAACzVJREFU/7+9uw+2q6rPOP59oBkiEOJrFWorGBCK8SIohYK2KorKSyhvBaVSBu20OFLBQqlVhyqIg+joyIz4jogyiLWiRAtEQQeIGCRI4tholUgHkBkoBZmo4SVP/9jrJDs3JxfJJGftfc7zmWFyz97n3PuQyZw7v7N+a/1ofh9d5+5+8j0b+F/WP8m1Uyt/xQ/Lh9rt/edd6+6QpANt31QeHEBzMmnEFtHrFTUASYey4crK++ol2lDZU/VQaUHYFtjB9r21c/WVpFttv6T9Sdtgf13tbBERMRnK7/PDgONp5qUuBC63fWPVYD0laRua1sKXlUs3AB/v0gBsSS8BPkfTcSSaw1lO7tiWmxgjvS7UJH0C2JZmmN9naI70XWL7zVWDtUg6cdh1218YdZZxIelm2/uX0x8/BtxD01Y6r3K0iIiYQJKeRjNM+ATbW9fO0yZpNs2R8tM/1D65WqiNKF1H2O50l4ykuQC2OzG7N8ZXr1sfgQNsT5WVlfdK+jDwn7VDTdNe5ZlNM6tqKZBCbdOdW94k/4lmPMMOwOl1I0VExKQp+7mPozkt9YfAX9dNNNSlwArgtcD7aFoK/6tqohY1k7jPBt5GaSOU9DhwYdc6pGD9Tq7BEPEu5ozx0PdCbTBX5TeSdqLpwd6xYp4N2D61/bicqnh5pThjwfbC8uVDNKupERERIyXpl8BtwBXAmR0+VXFX28dKOsL2JWWG4g21Q7WcTrNXdl/bK2Ht7LeLJJ1u+yNV07VsrJOraqgYa30v1BaWwucCmlUq080J8W2rgF1qh+iz0hrxd8DOtP4Nd7GNIyIixtaU7V/XDvF7GAyNflDSfOBemvEcXfEm4DW27x9csH2HpL8BrgU6U6jRj06uGCO9LtRsn1O+/KqkhcDsrvULS7qKdUc2bw3sSfPpW2y6r9N8GvhtoPNHI0dExFh6jqSv0cwfnC9pClhg+9zawab5VNlD926aERzbA++pG2k9s9pF2oDt+yTNqhFoBp3v5Irx0stCTdJfzHCPLkyxb/kQ6wq1x4A7bd9dMc842Nb2WbVDRETERPs0cCbwSQDby0pbYdcKtUuBo2m6UC4p155dLc2GZprV2qk5rvSzkyt6rJeFGs0b43QGpoA/plm5qqo1/FTTblnSauAXwLtsf2fk4fpvoaRDbH+rdpCIiJhY29peMjhQonisVpgZfJ1mT/etQGeOum/ZS9KwFlLROqWyC/rQyRXjpZeFmu3D248lHUizpH8vcOrQF43YTMNPJW0NzKcZ3Dx/ZKF6blrx+6+l4H20PLbtHWrmi4iIiXK/pHmUrhlJxwC/qhtpqOfafl3tEBvTtXEGw0g6aoZ7q4Ff2F4xwkgxIXpZqA1IOoimz9rAebYXVY70e7H9OHC7pAtrZ+mTdvFbhojvRsc+bYuIiIlxNk3b4x6S7gZWAl+uG2moxZJeZHt57SDDlN/nG2X7gVFlmcHhM9z7A+BPJS22/Y+jChSToZcDr8sMi3fRLOW/3/aNlSPFCEl6C/B24LnAj4D9gcW2D6oaLCIiJoakpcDfAnfQzP86DDjN9n5Vg00j6SfArjSF5GrWdaFMVQ1WSFrJ8K0i0OR8/ogjPWmStgKW235h7SwxXvpaqK0B7gJuZ91BHWvZXjDyUDEykpbTDBK/2faLJe1Bs6K60daEiIiIzanM+voK8Ebg5cCJwOFd27Mk6XnDrtu+c9RZ+k7Ss4HzgJ1sv17SnsCf2/6spB1td7H1NXqsr62PGXI82X5n+3eSkLSN7RWSdq8dKiIiJkeZ9fUG4Ergf4DX2v7tE7xs5PpUkJUxAutta+jYSd6fBy6m6eoC+BlNu+tnU6TFltDLQs3292pniKruKsfjXgkskvR/QG9+EUVERH+Vro52N8/TaU6b/kEZEdSJlsK+2ci2hu8Dr6qZa5pn2r5C0jsBbD8mKfNcY4vpZaE25E1y7S1gje29RhwpRsj2keXLf5N0PTAXuLpipIiImByH1Q4wpt7Oum0Nrxxsa6icabpVkp7BupM+96c5LyFii+hlocbwN0nRzFB754izREVZXY2IiFHqUythz/RhW8M7gG8A8yTdBDwLOKZupBhnvSzU2m+Skvam2ch7LM2JRl+tlSsiIiIiNknntzXYXirpL4HdaRYIfmr70cqxYoz19dTHFwBvKP/dT7OR8wzbQ082ioiIiIh+KMXQXOBq24/UztMm6QBgZ1qLHba/UC1QjLW+FmprgBuAN9v+ebl2Rx9mbURERETEhiS9DNjN9sWSngVsb3tl7VwDki4F5tEcdjI4RMQZdB1bSi9bH4GjgOOB6yVdDVzO8EGJEREREdFxks4GXkrTVngxMAv4InBgzVzTvBTY031c5Yhe2qp2gE1h+0rbxwN7ANcDpwF/KOkiSQfXTRcRERERT9KRwAJgFYDte4A5VRNt6MfAc2qHiMnR1xU1AGyvAi4DLitDEo8FzgKurRosIiIiIp6MR2xb0uDo++1qBxqQdBXNkfxzgJ9IWgKsHty3vaBWthhvvdyjFhERERHjQZKA9wB/BLwG+ABwMnCZ7QtrZoO1h5sIOB/45/Yt4Hzb+1UJFmMvhVpEREREVCVpOc2csoNpCqBrbC+qm2p9kpba3mfatWW2p2plivHW69bHiIiIiBgLS4EHbZ9ZO8h0kk4B3go8X9Ky1q05wE11UsUkyIpaRERERFQlaQWwK82Q61WD611YrZI0F3gaTUvmv7RuPWz7gTqpYhKkUIuIiIiIqiQ9b9h123eOOktEV6RQi4iIiIiI6JhezlGLiIiIiIgYZynUIiIiIiIiOiaFWkRE9IaknFYcERETIYVaRERUIWk7Sd+UdLukH0s6TtK+khaXa0skzZF0kqRvSLoO+E553efK/dskHVG+39aSLpB0i6Rlkv6+XH+FpO9K+ndJKyR9qQzY3ViuX0p6r6SlkpZL2qNc/zNJ3y8/c7Gk3cv1kyRdKWlRee3bJL2jPO9mSU8vz5sn6WpJt0q6YfB9IyIihsknkxERUcvrgHtsHwprj8C+DTjO9i2SdgB+W567DzBl+wFJ5wHX2T5Z0lOBJZK+DZwAPGR7X0nbADdJura8fm/ghcA9NHOPDgRunCHb/bb3kfRW4AzgLcAK4OW2H5P0auA84Ojy/PnlZ8wGfg6cZXtvSR8BTgQ+CnwK+Afb/y1pP+DjwKs2+W8vIiLGWgq1iIioZTnwYUnnAwuBB4Ff2b4FwPavAcri16LWvKKDgQWSziiPZwN/Uq5PSTqmXJ8L7AY8AiyxfVf5fj8CdmbmQu0/yp+3Ake1vt8lknYDDMxqPf962w8DD0t6CLiq9f84JWl74ADgK63FvG1m/NuJiIiJlkItIiKqsP0zSfsAhwDnAtfN8PRVra8FHG37p+0nlHbGU21fM+36K4DVrUuP88S//wbPbz/3HJqC7EhJOwPfHfJ8gDWtx2vK67cCHrT94if4uREREUD2qEVERCWSdgJ+Y/uLwAXAfsCOkvYt9+ds5PCQa4BTB/vMJO3dun6KpFnl+gskbbcZI88F7i5fn/RkXlhWB1dKOrZkk6S9NmO2iIgYM1lRi4iIWl4EXCBpDfAocArNatmFkp5Csz/t1UNedw7Nnq9lkrYCVgKHAZ+haWlcWoq4+4C/2ox5P0jT+vhu4Jub8PoTgIvK62cBlwO3b8Z8ERExRmS7doaIiIiIiIhoSetjREREREREx6T1MSIiJpKkrwG7TLt81vTDSCIiImpI62NERERERETHpPUxIiIiIiKiY1KoRUREREREdEwKtYiIiIiIiI5JoRYREREREdExKdQiIiIiIiI6JoVaREREREREx/w/YSpixLO68SEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": [],
      "needs_background": "light"
     }
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "gsu9Qs8eCtbJ",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1595407088150,
     "user_tz": -120,
     "elapsed": 566388,
     "user": {
      "displayName": "tizfa data",
      "photoUrl": "",
      "userId": "14479436635515447323"
     }
    },
    "outputId": "398e3bd0-6929-46cb-ca2a-752fe4d4ff89"
   },
   "source": [
    "plotErrorRatio(X_test_all, \"label_rand_forest\")"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>errors</th>\n",
       "      <th>num_samples</th>\n",
       "      <th>error_ratio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AINarendraModi</th>\n",
       "      <td>8</td>\n",
       "      <td>125</td>\n",
       "      <td>0.064000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeepDrumpf</th>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>0.379310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GenePark</th>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>0.181818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GenePark_GPT2</th>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>0.272727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gpt2Wint</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JustinTrudeau</th>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>0.019608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Musk_from_Mars</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thorin</th>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>0.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UtilityLimb</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VBoterin</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ahadsheriff</th>\n",
       "      <td>17</td>\n",
       "      <td>95</td>\n",
       "      <td>0.178947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ahadsheriffbot</th>\n",
       "      <td>36</td>\n",
       "      <td>95</td>\n",
       "      <td>0.378947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>awhalefact</th>\n",
       "      <td>4</td>\n",
       "      <td>35</td>\n",
       "      <td>0.114286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>botustrump</th>\n",
       "      <td>22</td>\n",
       "      <td>35</td>\n",
       "      <td>0.628571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calebgamman</th>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calebgamman2</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deep_potus</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deep_thorin</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dril</th>\n",
       "      <td>100</td>\n",
       "      <td>180</td>\n",
       "      <td>0.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dril_gpt2</th>\n",
       "      <td>44</td>\n",
       "      <td>179</td>\n",
       "      <td>0.245810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elonmusk</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt2_trump</th>\n",
       "      <td>44</td>\n",
       "      <td>55</td>\n",
       "      <td>0.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imranye</th>\n",
       "      <td>24</td>\n",
       "      <td>129</td>\n",
       "      <td>0.186047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imranyebot</th>\n",
       "      <td>43</td>\n",
       "      <td>129</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jaden</th>\n",
       "      <td>14</td>\n",
       "      <td>51</td>\n",
       "      <td>0.274510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kevinhooke</th>\n",
       "      <td>13</td>\n",
       "      <td>241</td>\n",
       "      <td>0.053942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kevinhookebot</th>\n",
       "      <td>1</td>\n",
       "      <td>241</td>\n",
       "      <td>0.004149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>narendramodi</th>\n",
       "      <td>11</td>\n",
       "      <td>124</td>\n",
       "      <td>0.088710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ninjasexparty</th>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0.391304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nsp_gpt2</th>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0.391304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>realDonaldTrump</th>\n",
       "      <td>14</td>\n",
       "      <td>129</td>\n",
       "      <td>0.108527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sarcastic_trump</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>theJadenTrudeau</th>\n",
       "      <td>30</td>\n",
       "      <td>103</td>\n",
       "      <td>0.291262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>utilitylimbgpt2</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whalefakes</th>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>0.057143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zawarbot</th>\n",
       "      <td>55</td>\n",
       "      <td>152</td>\n",
       "      <td>0.361842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zawvrk</th>\n",
       "      <td>34</td>\n",
       "      <td>152</td>\n",
       "      <td>0.223684</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 errors  num_samples  error_ratio\n",
       "screen_name                                      \n",
       "AINarendraModi        8          125     0.064000\n",
       "DeepDrumpf           11           29     0.379310\n",
       "GenePark              4           22     0.181818\n",
       "GenePark_GPT2         6           22     0.272727\n",
       "Gpt2Wint              1            2     0.500000\n",
       "JustinTrudeau         1           51     0.019608\n",
       "Musk_from_Mars        1            2     0.500000\n",
       "Thorin                3           10     0.300000\n",
       "UtilityLimb           3            4     0.750000\n",
       "VBoterin              1           13     0.076923\n",
       "ahadsheriff          17           95     0.178947\n",
       "ahadsheriffbot       36           95     0.378947\n",
       "awhalefact            4           35     0.114286\n",
       "botustrump           22           35     0.628571\n",
       "calebgamman           6           13     0.461538\n",
       "calebgamman2          1           13     0.076923\n",
       "deep_potus            2            3     0.666667\n",
       "deep_thorin           5           10     0.500000\n",
       "dril                100          180     0.555556\n",
       "dril_gpt2            44          179     0.245810\n",
       "elonmusk              1            6     0.166667\n",
       "gpt2_trump           44           55     0.800000\n",
       "imranye              24          129     0.186047\n",
       "imranyebot           43          129     0.333333\n",
       "jaden                14           51     0.274510\n",
       "kevinhooke           13          241     0.053942\n",
       "kevinhookebot         1          241     0.004149\n",
       "narendramodi         11          124     0.088710\n",
       "ninjasexparty         9           23     0.391304\n",
       "nsp_gpt2              9           23     0.391304\n",
       "realDonaldTrump      14          129     0.108527\n",
       "sarcastic_trump       1            5     0.200000\n",
       "theJadenTrudeau      30          103     0.291262\n",
       "utilitylimbgpt2       1            4     0.250000\n",
       "whalefakes            2           35     0.057143\n",
       "zawarbot             55          152     0.361842\n",
       "zawvrk               34          152     0.223684"
      ]
     },
     "metadata": {
      "tags": []
     },
     "execution_count": 31
    },
    {
     "output_type": "display_data",
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2oAAAGICAYAAAAwOBOrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhlZXX3/e+PSZxQEzqOTBrUEMWoLc5xzotBITGi4MyjYqKo0egTjAYN5okYo8ZZcTYOiBoVFYMRB5wQGlEQkIiIAmpE40AcQdb7x71P96nqqq4Cq2vfp/v7ua6+qvY+u6tWd9U5Z697WCtVhSRJkiSpH9uMHYAkSZIkaS4TNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM5sN9Y33nnnnWv33Xcf69tLkiRJ0qhOO+20H1TVmoUeGy1R23333Vm3bt1Y316SJEmSRpXkW4s95tJHSZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTPLStSS7Jvk3CTnJTl8gcd3TfLJJKcnOSPJn658qJIkSZK0dVgyUUuyLfAq4P7AXsDBSfaad9lzgGOr6rbAQcCrVzpQSZIkSdpaLGdGbR/gvKo6v6p+DRwDHDDvmgJ2Gj6/DvCdlQtRkiRJkrYu2y3jmhsDF04dXwTccd41zwM+luTJwDWB+65IdJIkSZK0FVpOorYcBwNvqaoXJ7kz8G9JblVVV0xflORQ4FCAXXfddYW+tSRpvt0P/8iKf80Ljtpvxb+mJEla2HKWPl4M7DJ1fJPh3LTHAscCVNUXgB2Bned/oao6uqrWVtXaNWvWXLWIJUmSJGkLt5xE7VRgzyR7JNmBVizkuHnXfBu4D0CSP6AlapesZKCSJEmStLVYMlGrqsuBw4ATgHNo1R3PSnJkkv2Hy/4GeHySrwDvAh5TVbW5gpYkSZKkLdmy9qhV1fHA8fPOHTH1+dnAXVc2NEmSJEnaOi2r4bUkSZIkafWYqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOrOsRC3JvknOTXJeksMXePylSb48/PmvJD9e+VAlSZIkaeuw3VIXJNkWeBVwP+Ai4NQkx1XV2ZNrquppU9c/GbjtZohVkiRJkrYKy5lR2wc4r6rOr6pfA8cAB2zi+oOBd61EcJIkSZK0NVpOonZj4MKp44uGcxtJshuwB/CJ3z40SZIkSdo6rXQxkYOA91bVbxZ6MMmhSdYlWXfJJZes8LeWJEmSpC3DchK1i4Fdpo5vMpxbyEFsYtljVR1dVWurau2aNWuWH6UkSZIkbUWWk6idCuyZZI8kO9CSsePmX5TklsD1gC+sbIiSJEmStHVZMlGrqsuBw4ATgHOAY6vqrCRHJtl/6tKDgGOqqjZPqJIkSZK0dViyPD9AVR0PHD/v3BHzjp+3cmFJkiRJ0tZrpYuJSJIkSZJ+SyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktSZ7cYOQNLmt/vhH1nxr3nBUfut+NeUJElS44yaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ5aVqCXZN8m5Sc5Lcvgi1zwkydlJzkryzpUNU5IkSZK2HkuW50+yLfAq4H7ARcCpSY6rqrOnrtkTeBZw16r6UZLf21wBS5IkSdKWbjkzavsA51XV+VX1a+AY4IB51zweeFVV/Qigqr6/smFKkiRJ0tZjOYnajYELp44vGs5Nuzlw8ySfS3Jykn0X+kJJDk2yLsm6Sy655KpFLEmSJElbuJUqJrIdsCdwT+Bg4PVJrjv/oqo6uqrWVtXaNWvWrNC3liRJkqQty3IStYuBXaaObzKcm3YRcFxVXVZV3wT+i5a4SZIkSZKupOUkaqcCeybZI8kOwEHAcfOu+QBtNo0kO9OWQp6/gnFKkiRJ0lZjyUStqi4HDgNOAM4Bjq2qs5IcmWT/4bITgB8mORv4JPDMqvrh5gpakiRJkrZkS5bnB6iq44Hj5507YurzAp4+/JEkSZIk/RZWqpiIJEmSJGmFmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTObDd2AJI0sfvhH1nxr3nBUfut+NeUJEna3JxRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6syyErUk+yY5N8l5SQ5f4PHHJLkkyZeHP49b+VAlSZIkaeuw3VIXJNkWeBVwP+Ai4NQkx1XV2fMufXdVHbYZYpQkSZKkrcpyZtT2Ac6rqvOr6tfAMcABmzcsSZIkSdp6LSdRuzFw4dTxRcO5+f4iyRlJ3ptklxWJTpIkSZK2QitVTORDwO5VtTfwn8BbF7ooyaFJ1iVZd8kll6zQt5YkSZKkLctyErWLgekZspsM59arqh9W1a+GwzcAt1/oC1XV0VW1tqrWrlmz5qrEK0mSJElbvOUkaqcCeybZI8kOwEHAcdMXJLnh1OH+wDkrF6IkSZIkbV2WrPpYVZcnOQw4AdgWeFNVnZXkSGBdVR0HPCXJ/sDlwP8Aj9mMMUuSJEnSFm3JRA2gqo4Hjp937oipz58FPGtlQ5MkSZKkrdNKFRORJEmSJK0QEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpM8tK1JLsm+TcJOclOXwT1/1FkkqyduVClCRJkqSty5KJWpJtgVcB9wf2Ag5OstcC110beCrwxZUOUpIkSZK2JsuZUdsHOK+qzq+qXwPHAAcscN3zgRcCv1zB+CRJkiRpq7OcRO3GwIVTxxcN59ZLcjtgl6r6yKa+UJJDk6xLsu6SSy650sFKkiRJ0tbgty4mkmQb4CXA3yx1bVUdXVVrq2rtmjVrfttvLUmSJElbpOUkahcDu0wd32Q4N3Ft4FbAp5JcANwJOM6CIpIkSZJ01SwnUTsV2DPJHkl2AA4Cjps8WFU/qaqdq2r3qtodOBnYv6rWbZaIJUmSJGkLt2SiVlWXA4cBJwDnAMdW1VlJjkyy/+YOUJIkSZK2Ntst56KqOh44ft65Ixa59p6/fViSJEmStPX6rYuJSJIkSZJW1rJm1KTVtvvhm+z0cJVccNR+K/41JUmSpM3BGTVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnbGYiCRJkqQVZ3G4344zapIkSZLUGRM1SZIkSeqMiZokSZIkdcZETZIkSZI6Y6ImSZIkSZ0xUZMkSZKkzpioSZIkSVJnTNQkSZIkqTMmapIkSZLUme3GDkCSpN7tfvhHVvxrXnDUfiv+NSVJWw5n1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnVlWopZk3yTnJjkvyeELPP6XSc5M8uUkn02y18qHKkmSJElbhyUTtSTbAq8C7g/sBRy8QCL2zqq6dVX9EfDPwEtWPFJJkiRJ2kosZ0ZtH+C8qjq/qn4NHAMcMH1BVf106vCaQK1ciJIkSZK0ddluGdfcGLhw6vgi4I7zL0ryJODpwA7AvRf6QkkOBQ4F2HXXXa9srJIkSZK0VVixYiJV9aqquhnwt8BzFrnm6KpaW1Vr16xZs1LfWpIkSZK2KMtJ1C4Gdpk6vslwbjHHAH/22wQlSZIkSVuz5Sx9PBXYM8ketATtIOBh0xck2bOqvj4c7gd8HUmSJGmG7H74R1b8a15w1H4r/jW1dVgyUauqy5McBpwAbAu8qarOSnIksK6qjgMOS3Jf4DLgR8CjN2fQkiRJkrQlW86MGlV1PHD8vHNHTH3+1BWOS5IkSZK2WitWTESSJEmStDJM1CRJkiSpMyZqkiRJktSZZe1RkyRJ/bNinSRtOZxRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnbHhtSRdCTYUliRJq8EZNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZyzPL0mSJGmr1WvrHWfUJEmSJKkzJmqSJEmS1BkTNUmSJEnqzLIStST7Jjk3yXlJDl/g8acnOTvJGUlOTLLbyocqSZIkSVuHJRO1JNsCrwLuD+wFHJxkr3mXnQ6sraq9gfcC/7zSgUqSJEnS1mI5M2r7AOdV1flV9WvgGOCA6Quq6pNV9fPh8GTgJisbpiRJkiRtPZaTqN0YuHDq+KLh3GIeC3z0twlKkiRJkrZmK9pHLckjgLXAPRZ5/FDgUIBdd911Jb+1JEmSJG0xljOjdjGwy9TxTYZzcyS5L/BsYP+q+tVCX6iqjq6qtVW1ds2aNVclXkmSJEna4i0nUTsV2DPJHkl2AA4Cjpu+IMltgdfRkrTvr3yYkiRJkrT1WDJRq6rLgcOAE4BzgGOr6qwkRybZf7jsRcC1gPck+XKS4xb5cpIkSZKkJSxrj1pVHQ8cP+/cEVOf33eF45IkbQV2P/wjK/41LzhqvxX/mpIkrbZlNbyWJEmSJK0eEzVJkiRJ6syKludX/1xmJEmSJPXPGTVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjpjoiZJkiRJnTFRkyRJkqTOmKhJkiRJUmdM1CRJkiSpMyZqkiRJktQZEzVJkiRJ6oyJmiRJkiR1xkRNkiRJkjqz3dgBSLNs98M/suJf84Kj9lvxrylJkqTZ4oyaJEmSJHVmWTNqSfYFXgZsC7yhqo6a9/gfA/8K7A0cVFXvXelAJUmSNJtcgSJdeUvOqCXZFngVcH9gL+DgJHvNu+zbwGOAd650gJIkSZK0tVnOjNo+wHlVdT5AkmOAA4CzJxdU1QXDY1dshhglSZIkaauynD1qNwYunDq+aDh3pSU5NMm6JOsuueSSq/IlJEmSJGmLt6rFRKrq6KpaW1Vr16xZs5rfWpIkSZJmxnIStYuBXaaObzKckyRJkiRtBstJ1E4F9kyyR5IdgIOA4zZvWJIkSZK09VqymEhVXZ7kMOAEWnn+N1XVWUmOBNZV1XFJ7gC8H7ge8MAk/1BVf7hZI5ckSdrKWfZe2nItq49aVR0PHD/v3BFTn59KWxIpSZIkSfotrWoxEUmSJEnS0kzUJEmSJKkzJmqSJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOrOs8vxjW+keIfYHkSRJktQzZ9QkSZIkqTMzMaMmSZIkqVnp1WbgirMemahJkqRV5U2mJC3NpY+SJEmS1BkTNUmSJEnqjImaJEmSJHXGRE2SJEmSOmOiJkmSJEmdMVGTJEmSpM6YqEmSJElSZ0zUJEmSJKkzJmqSJEmS1BkTNUmSJEnqzHZjB7Cl2P3wj6z417zgqP1W/GtKkiRJ6p8zapIkSZLUGRM1SZIkSerMshK1JPsmOTfJeUkOX+DxqyV59/D4F5PsvtKBSpIkSdLWYslELcm2wKuA+wN7AQcn2WveZY8FflRVvw+8FHjhSgcqSZIkSVuL5RQT2Qc4r6rOB0hyDHAAcPbUNQcAzxs+fy/wyiSpqlrBWCVJklaNhcIkjSlL5VJJHgzsW1WPG44fCdyxqg6buuarwzUXDcffGK75wbyvdShw6HB4C+DclfqHDHYGfrDkVeMzzpU1C3HOQoxgnCvNOFeWca6cWYgRjHOlGefKmoU4ZyFG2Lrj3K2q1iz0wKqW56+qo4GjN9fXT7KuqtZurq+/UoxzZc1CnLMQIxjnSjPOlWWcK2cWYgTjXGnGubJmIc5ZiBGMczHLKSZyMbDL1PFNhnMLXpNkO+A6wA9XIkBJkiRJ2tosJ1E7FdgzyR5JdgAOAo6bd81xwKOHzx8MfML9aZIkSZJ01Sy59LGqLk9yGHACsC3wpqo6K8mRwLqqOg54I/BvSc4D/oeWzI1hsy2rXGHGubJmIc5ZiBGMc6UZ58oyzpUzCzGCca4041xZsxDnLMQIxrmgJYuJSJIkSZJW17IaXkuSJEmSVo+JmiRJkiR1xkRNkiRJkjpjoiaS7DF2DJI0y5JcbTnnJElarpksJpLk3lX1iSQPWujxqvr31Y5pMUk+CWz0n1xV9x4hnAUlOa2qbp/kxKq6z9jxLGWhOHuMPckLq+pvlzrXgyS3A+5G+139XFV9aeSQ1lvseT7R0/MdIMn2VXXZvHM7V9UPxoppMcP/7eTn/tmqev/IIW0kSYCHAzetqiOT7ArcoKpOGTm0OZJ8qaput9S5sQw/6xcCvwdk+FNVtdOogS0gyc2B1wDXr6pbJdkb2L+q/nHk0OZIcmBVvWepc2NJ8oiqenuSpy/0eFW9ZLVj2pQkvws8D7grw2sScGRVddOXN8mjFjpfVW9b7Vg2JcldgS9X1c+SPAK4HfCyqvrWyKHNkeSpVfWypc6NLcmRVXXE1PG2wNuq6uGb+3svWZ6/U/cAPgE8cIHHCujpxu0ZU5/vCPwFcPlIsSxmmyR/B9x8oRf0Xl7Mk+wIXAPYOcn1aDcaADsBNx4tsMXdD5iflN1/gXOjSnIEcCAbnjdvTvKejm6KFnqeT3TzfE9yL+DfgB2TfAk4tKouGB7+GO2NshtJXg38PvCu4dQTkty3qp40YlgLeTVwBXBv4EjgUuB9wB3GDGoiyQ1orz9XT3Jb5r4uXWO0wDb2z8ADq+qcsQNZhtcDzwReB1BVZyR5J9DLa9LEs4D5SdlC58ZyzeHjtUeNYvmOAU6i3SdBG6B5N3Df0SLa2PTrzo7AfYAvAV0larSBjtskuQ3wN8AbaDHeY9SoNvZoYH5S9pgFzo1tlyTPqqoXDCsljgVOX41vPJMzarMuySlVtc/YcUwkuQXwZ8BfA6+d/3hV/cOqB7WAJE+lxXgj4DtTD/0UeH1VvXKUwOZJ8lfAE4GbAedNPXRt4POrMQJzZSQ5F7hNVf1yOL46bSTuFuNGNluSnAo8Zugz+WDgBcAjq+rkJKdX1W1HDnGOJF8D/qCGN4Ek2wBnVdUfjBvZXJNZqen/wyRfqarbjB0bQJJH024s1gLrph66FHhLLzO+ST5XVXcdO47lSHJqVd1h3s/8y1X1R2PHBpDk/sCfAg+hJRITOwF79fT+PkuSfLWqbjXv3JlVdeuxYlpKkusCx1TVvmPHMm3qdfMI4OKqemNnM/wHAw+jrej4zNRD1wau6HCFVIB3AGcC9wKOr6p/XY3vPZMzaotN40/0MgMEkOR3pg63AW4PXGekcBazb1W9MMnVqurIsYNZzDAV/rIkT66qV4wdzya8E/go7Ub98Knzl1bV/4wT0iZ9hzYy+Mvh+GrAxeOFM9cMLd/ZoarOAqiq9yY5B/j3JH/LAsufO3AesCswWQqzC3MHFnpx2bDMZJJQrqHNsHWhqt4KvDXJX1TV+8aOZxPWJXk38AHgV5OTvSSS8/wgyc3Y8DN/MPDdcUOa4zu0pHx/4LSp85cCTxslogUkefmmHq+qp6xWLMv0sSQH0WYrAB4MnDBiPMvxM6DHff6XJnkW8Ejg7sNA3PYjxzTt87Tn9M7Ai6fOXwqcMUpECxi2hUy8jDbL/zngpCS3W41tIjM5o5bkucOnt6BNQx83HD8QOKWqHjFKYAtI8k3am01oSx6/SVtz/dlRA5syGansabRlU5LsAPwl8MfDqU8Br5u/L2hsSe5Em6G4dDjeiTaD8cVxI5sryQdoz6P/pP2u3g84BbgIxn8zT/KEqnrd1PN+jo5mfNcBD6iq702duwnwYeBmVdXV8qMkn6b93Cd7ve5Au/n8CUBV7T9SaHMkeTjwUNrS0bfSbt6e08s+oIlhZP0INrwufZr2Wv+T8aLaIMmbFzhdVfV/Vj2YJSS5KXA0cBfgR7T3zYd3uL9me9p7+82HU+f29D40zPZC2/O1Fxtm/w4Ezq6qvxwlsEUkuZS2XHMyELMNLRGCTvZTJvkQGwbetqH9vx5bVYcv/rdW37Ak+2HAqVX1mWFv7z1720sHkOT6bFhSekpVfX/MeKYNdSYWU6tRb2ImE7WJJCcB+03dCF8b+EhV/fGm/6amJXkXbdnOjYBvTD9E+0Xce5TAFpHkDbSRobcOpx4J/KaqHjdeVBtLcjpwu3lLy9b1lgxPvZkvaJgxGNUwo/KUqnrp2LEsJsl9gUuq6ivzzl8HOKyq/t84kS0sySb3KlTVp1crlqUkuSVtL0iAE3vcZ5XkfcBXmfu6dJuq2mQxHG0sye2r6rQk1wS2qapLkzygqj48dmzThufQ24ALaL+buwCPrqqTxoxrviQnA3erqsuH4+2Bz1TVncaNbPbMe928HPhWVV00VjybkmQ3YM+q+niSawDbTu6Xe5HkQOBfaAPuAe4OPLOq3jtmXNOGe7cDq+rdS168Ob7/jCdq5wJ7V9WvhuOrAWf0trcmya1ooy47Ts71NqoxjL6cQFvKMUeHo5gb7U/pac/KxEJ7KpKc0VviOyt629u5mFmpYjUxzPSuXwbf2/LcJC8G3lhVZ48dy6Ys8nzvaV/VTFRShLa/BnhUVX11OD4IeFpV3XHcyOZKchrwsKo6dzi+OfCuqrr9uJHNNdwr3Xny3E4rxnVyb/dKAMPv5e7MfU3qcXlu15I8HjgU+J2qulmSPYHXdrj36yvA/SazaMPS9o93eD+3rqrWjvG9Z3KP2pS3AackeT8tEz8AeMuoEc0zLNe6Jy1RO55W9e+zdFYhaFiu1dUTYxN+k+RmVfUNWL9M5jcjx7SQ85M8hXZzBK3AyPkjxrOgJA8Ang/sRntN6LVs9+eSvJK2fGeyHIbVWCN+Jc1EFaskh9KqKP6SttQotCU9Nx0zrgWcA7w+yXbAm2k3wl0sJ5znF0nuNlnWnlYe+xcjxzRtViopQlve+t4kD6ONsD8K+JNxQ1rQ9pMkDaCq/muYrerNUcDpwzKu0JbnPm/UiBaQ5E3A3sBZbFj+2E1lX1i/peEVwB8AOwDbAj/r8P3yScA+wBcBqurrSX5v3JAWtM28pY4/pM8ezx9P8gw2vv/Y7AObMz2jBus3+t2d9mT+TFWtSrnM5UpyJi0BOr2qbjOsxX17Vd1v5NDWS3JsVT1kiHX6F6LXpY/3od2wnU+LcTfgkKra1FriVTe8KL6cVla8gBOBv+5p/TVAkvOABwFnVscvCFNrxScxTn4/u+hJmNmrYvV12ih7d/3dFpJWnfYQ4GDaZu7X9/ScT/JHtGWP16H9bv4PbRlcFxvj03klxfmG2akPAN8G/ryqekp6gfWJxRXA24dTD6ctL+tx398NgDvSXj9Pmd5L24skZ1fVXmPHsSnDXuSDaC0Y1tIGEW5eVc8aNbB5knyxqu44eb4PA11f6vB+7kW05HzSJuahtJVxvbUx+uYCp6uqNvvA5qzPqEGbSbmC9uLTTSWwKb+oqiuSXD4sMfo+bR17T546fHzAqFEsU1WdOEzjT5ZtnDtZ/tqTISE7aOw4luFC4Ku9JmnZUO3xw2wozDPRU8wzUcVqyjeAn48dxHIMexRvOfz5AfAV4OlDoZkunmNV9WVa36KdhuOfjhzSfL1XUmSBwcLfoc1YfDEJvd1kAn9Fm7mYFFz6DK3vX4/2oQ1qQ/s//tCIsSzmC0n26n2Zc1Wdl2TbqvoNre/o6bT+eT35dFp/3KsnuR9tRU93P/OqemaSB9EGOAGOrqr3jxnTQqpqtMqeM52opfXVejyt+WmAtyc5uvoq3b4urRrY62llfP8X+MK4Ic1VVd8dboTeUlX3GjuepaQ1vn4i7YldwGeSvLaGPmC9GOJ8LPCHzN2f2Nto6/8Fjk+rAjhdtruXsveTaomTKq8fpD3fH8iGioWjG/Zyfgu48zB6vQ/t9/PcySb+zjwL+HySLzL3595Vye4kL6X9rE8E/qmqJj/zFw57b7qQ5HeB5zK8LiX5LK3q4w/HjWy9J9EqKd4yycW0SordVEgezMRg4URV/WpYjn0ibaD43Kr69chhbSTJUbTXzncMp56S5M5V9XcjhrWQt9GSte/RXpN6XNXz87TK019O8s+0wY4el+odTrv/OBN4Aq3v1+vHDWlRn2fDpMupI8eyoCRn0Gb93l1Vq7qFZaaXPg7/cXeuqp8Nx9cEvtDZk3q9JLsDO/WyFGa+JCcCD+p0/8d6SY6lzVJMlps8DLhuVR04XlQbS/Ie4Gu0+I6kLYs5p6qeusm/uMqSfIw2gHAmU7PS1UnZ+4nMSJXXJI+l3bB/gnajcQ/aDfubRg1sniSn0PbLzv+5j17lc1qSQ2jlr3+2wGPX6eX1Ksl/AicxdxncPavqvuNFtbFMVVIcO5ZNSXIbNswAfabmVVPtQZL9gNfSZqdD66f1hKr66KiBzTPcK/1RVV0xHG9L247R1b3SsAz/6Wz8mtRNQbO0Sor/Tduf9jTaUtmW5K8AACAASURBVOdXV1VXPSiTHFlVR0wdbwu8raoePmJYG0nyOFpbk97fL3ejLct8KO13892096Vvb/bvPeOJ2pnAHSYzKcMMxqnVURf7JKG9Yd+0qo5M62Vxg6lR4W4k+SBwW1o/renNkr2NsG+0jr3Hte1Ta8PPqKq902lJ5CRfrapbjR3HUjI7VV7PBe4ymUkZZlo+32Gc6/cq9S7JjdlQ7AaA6q8E+kbPoyRnjv1+lEUaxU90NHO+3tRqmUkRiT+nLYnqabUMSb5G65143nB8M9rg0S3HjWyuIVG7Z22o+vg7wKc6TNS+UFV3HjuOpSS5OrDrdCGZ3qT1TfyvqnrBMAN4LPDlqnreuJHNNSvvl9OGrTd/T+vtuO3m/n4zvfSRVlDii2lVHwH+DHjjiPEs5NW07PvetFmVS2lLNe+wqb80kn+no+pKm/ClJHeqqpMBktyR1qi3N5PGpz9Oa9HwPaDHqkvHJ/mTqvrY2IEsYbrKK7Tn+1vGC2dRP6Q9zycuHc715qNplR8/xNylj72V5z+KttfzbDZUdy3a7FVPPpZWRv7Y4fjBtJYnY5u/dPi44birpcPzPBa449RqmRfStgx0lagBl86bSTmfuc/9XryAjas+dtWgeXB6WiXS+a9J3dyXJHkgre/XDsAeQxGhI6tqo9ZGI/s/wDuSPAu4F/DR6rMP6ay8X86fVfsNbdvI5v++szyjBuurPk42IfZY9fFLVXW7zK201V3Pr4meR4qmNppvT7vp+PZwvBvwtQ5n1B5HS8pvTUsorgX8fVW9bsy45ktyKXBN2hvjZfRbnn+6yivASb093wGSvI32M/8g7ffzAFoxkTOgnxmMMatYXRnzZ1J7Mzx/JkVursmG4lbbAv/by/NoVpYOQ/+rZYbiBwD3o73/HEv7mR8IfLuqnjhWbItJckM2DBD3WvXxzQucrp72daf1zrs3bUZyck83+sz5xPAeObE9rR3H5xgmMaqTdjZTM/1/xALvl1X1mJFCW9Cwl3t7WrXPVd2nNpMzasO0/cQFw5/1j3U2InzZsDZ4UmlrDX1Wp5yFkaID6LNf2hzZ0Nz4nKr6EW3kv6ub32lVde2lr+rD8CbTxRvNJnxj+DPxweFjV//PY1axupLOp71BdpmozdDz5/rAdKGLXw/nejS9WmbSI7Wn1TIPnPr8v2n7agAuYapwVGfWDB+3A+6SVkWzm5kqgKo6ZOwYluGyqvpJ29WyXk8zHi+ed/wjWh/fF9Pi7KKdDRveDxd7v+zNI6vqv8b4xjM5o5bkCuAiYFJJbU657p5GhJM8nDZNejtaj50HA8+pqveMGtgCFhkp6mb/UpLTqur2SU6sznpSTcvQm2gymzp2PEtJsuCIem97gLSykjxqofNV9bbVjmVTkryP1ovyRPquTjnZj7xHVT0/yS7ADXvZj5zk2cBDgOmlw8dW1T+NF9XiplbLFPDZHmfPZ0UWaSTd00wVrJ9R2+imtKc4k7yR9lp0OPAXtNYM21fVX44a2AwaJjFeWFXPGDuWpST5BnAyrQXHZ6rqrNX63jM5o0ZrInwv2nTuu2gv4l1mnFX1jiEBug8tofyzqjpn5LAWs9BIUU+zf9uk9QW5+UIb5HtZUgack9ZM+EbDJu6JHksNAzxz6vMdaWXlJ0m7rqQkH2Ljm42f0PZRvq76aSMxvU92R9pr1JdoewF7chwb9lX1bHo/8vNplVRfRSf7kavq/yX5DzZsFTik8+TnN7TnUa89UidNuV8DXL+qbpVkb2D/qvrHkUOb7069bQ1YxIenPt+RVkTmOyPFspgnA8+mDRq9i7YP9fmjRrSIoSrp/PZAR44X0VxV9Zskdx07jmXai9Yw/u7Ai5LcgrZE88839zeeyUStqv56GL28J/BI4BVpJcZfU1UL7btYdfOWZ36fDV3Xe1yeOXFWkocB2w5VbZ5C62/Ri4Noo8Db0dkysmlVdXBaH60TgF6WjS6qqqaX8TDMBPzrSOFsCc6nLTOaPOcfStsgfXNaP8VHjhTXHFX15OnjtH6Px4wUzqKqs3YBm3DHyX5kgKr60VBtrRtVdVqSCxlu3JLsWqtQXvrKymz0SIX2fH4mbR8QVXXGUAyjt0RtVhpJv2/6OMm7aC1EulFVP6clas8eO5ZNSfJa4Bq0SY030FZzdTG7P8+XkxxH2/s1XW28q2W5tIGjy9jQ7+37w5/NbiYTNWjTEsAnhzfFg2gjGl+nvXD24DQ2bDDflbZOOMB1aUUwetwf0vtI0b5V9cIkV+tpVGghw0bt2/RcnGUTLgL+YOwgZthdqmp6FuVDSU6tqjskWbXlElfBz+jwdWkYNHoBbURzemS4myXug673IyfZn7ZP5Ua0G4xdaX0e/3DMuBYxK1Ufr1FVp8xbhdJjc/tZaCS9kD3ppFLyIisl1utoL//EXaq1BTqjqv4hyYuBrvr7DXakVXmcXsFT9FeB/Ke0/n4vAV4/aSewGmYyUUtr2HkAbaR6De0HevueRgYnG/WTvB54f1UdPxzfnzYr1J0ZGCk6BHgZ7f+v60QNZqI4CwBJXsGGN6BtaFWYei/Y0bNrTc9UpPVOvNbw2K8X/2ura96Nxza0ROjYxf/GaN5MayD+Utro8CG0eHvzctr+r99L8v8Y9iOPG9IczwfuBHy8Wn/HewGPGDmmxYS5haN+w9y96L34QVrvtEly/mDgu+OGtKA30mby5zSS7s1UBdWJ7wF/O1I48/3L8PFBwA3Y0Nj+YFpBmd5Mltj/PMmNaMnQDUeMZ0EzUkAG2s/5bsATgccl+Tyt8vSJm/sbz2oxkZ/RZs+OGT7O+Uf0NGW6UNnWnkq5ThvW2z8D2J25jWW72Ks0LINYSxsRnq4S1OXoYO9lfCeSPHrq8HLggqr63FjxzLokfwq8lvY7Gtos1ROBTwGPr6oulpUmucfU4eXAt6rqorHiWcxUEaH1z53JubFjmy/JLdmwH/nEnvYjJ1lXVWuTfAW4bVVdkU5bxQx7kB/N3MInb+nluTOR5KbA0cBdaKtmvklrgvutUQObJzPQSHrYzrJLTwPuC5k8j5Y6N7Ykf0+bgb4Pba9s0WaCjhg1sHnSWm88lo330nVTQGba8Bp/f+Cvgd+rqqtv7u85kzNqtLWsReulNb97eW9Tpt9J8hw2jL48nP42x068h3aD+QY6LIM/a3u/6L+M76Tq0p9U1cPHjmVLUVXHD8v1bjmcOneqgMi/JrlfVf3nSOEB63/uz6uqe40ZxzL9Ksk2wNeTHAZczIYZym4MS4veWFWvGjuWRfw4ybVoVcvekeT7TO0J6UlVvSTJp4FJoYFeC59cr6ruO6zy2aaqLk3yAKCrRI0ZaCRdVZXkI7SeWj27ZpKb1tBHK8ketP6JXUhy4FBV/O1V9WPgfUk+DOxYVT8ZObyF/BttCfb/R1sp9XCgmwGuianqw9+gtVx6FPDFVfneszijNkuGoiLPBSYl0E8C/qHHYiK9jlLPqlkp45vks8C9q6qbZXlbsnTStiHJicCDOn3zXi/JHWhv3NelLd/bCXhRVZ08amDzpDW4P4Q2APpm4F09/d8muQZtOVRoSx53At7R43sRrB9MuD5zV3d0NduS5EvAo6rqq8PxQcDTquqO40Y2V2agkTRAkrcCr6yqU8eOZTFJ9qXNop5Pey7tBjyhqk4YNbDB5P2ll/eZpSQ5fViKfcawp257Wvn7O40d27Qka4HTq2rVJzFmPlHrvfzoLJiqUPkU2ibz9zN31K2LN/Ikx1bVQ5KcydyZqV6XPl6Dtt/vT2gxngA8v6Py7AAkeRuteMhxzK261Eu7gy3K5I2pgzg+CNwW+E/m/ty76U+WGeqzM5FWtvkQ2p6Gz9GWG31yxHg+W1V3m7f/ZzLNfwXwP7TE99WjBLiAJE+mDXD+Nxv2p/X4Gn9T4L3Aw2hlux8FPKCnBH0WJDmsql6Z5GvA79NmJH9Gvz/3q7FhxcTXqupXm7p+NSX5T9rz/A602fM5Otwjf0pV7ZPkJNoWge8Bp3RYLIokt2LjolabvZ3NTCdqi5UfrarHjhrYlCSfZOEGjl3s+wJI8k02VKicr3p5wiS5YVV9N8luCz3e276AWZHkuQucLgc8No9eRjrn7U2cqNV447kykpzc2+jqYobE8gG0RG0XWnGWuwE/q6qDxoxtMUl+F/h8Vc3fRjCaJOfRqj6uWmW1q2rY2/0BWjXnP6+qX4wc0kZ63wc0NQvU/Xt7kkctdL6X1820liC3oy0pfNz8x6vq06se1CYMKxHeR2vI/mbasva/r6rXjRrYPMN90j1pidrxtH1qn62qB2/u7z2re9QmZqH86PRI8I60JXBdle+toUJl74YkbVvapvJu99Zk9sr4nj2saV8vyYFjBaNVc92qetn0ibT+Vb05PTPQZyfJS4EH0pY7/1NVTXoWvTBJt+05quqHSe45dhzzXEhrEt+lBVZ1/A6wLfDFJPQ2A8SM7APqKSHbhOnWKzvSinV8idYCYXTDFoaTk9ylqi5Jco1qFb27VFVvGD79NNDFpMAiHkzbo3Z6VR2S5PpsqD2xWc16otZ9+dGqOm3eqc8l6bHp4OTm/D+GDdHPoY3KPL+nTdzVOtlfkeQ6HS8v+ZelL+nKs2g3wUud08q4YOwABo+mtbuY9pgFzo1tVvrsnAE8p4beX/Pss9rBXBlV1UVJ+aHaI7T9P58aiktML8PvZTn2A8YO4Er6/ao6MMkBVfXWobDIRsviRrR3kp8ucH6y9HGn1Q5oMVX15OnjJNelVSDvze8PBXmuBeya5Da0vXRPHDmuOZJ8AziZ9vv4marqtdfoL6tVyr08yU5s6EW52c16ovah4UnyItqIRtFPw2tgzv4vaL1/bg9cZ6RwlvL3VfWeJHcD7kv7f30t0NXGaOB/gTOHtdjd7a3pbWnBYtJ6+v0pcOMkL596aCc6m/WdBUketKnHJzNAVbXJ6za3JAfT9tTsMcxUTVybtl+pKzUjfXaq6s1JbjzcEE0XwDip40Gl3lx7+Pjt4c8Ow5+uTM/8DD/vuw+Hn6mqr4wT1SZdNnz88bDP5nt00kh6cGYP+3avop/RWrD05l9pM6jHAVTVV5L88ab/yij2ot1j3h140bDH94yq+vNxw9rIqUO+8XrgNNp96Kq0MZrZRC2tXPOJM1B+9DQ27P+6nNZnpZs9dPNMqtnsBxxdVR9J8o9jBrSIf6e/0fSNpJVofwEbbz7tZXr/O8A6WquD6ZnfS4GnjRLRbHvgJh7raQbo87SmvDsDL546fyltVqgrQ/nrJ7Nxf8eulhAnOQo4CDibDa+lRav0q2Woqn8YO4YrY1gq/Hg2PLffnuToqnrFiGEt5Ogk16M1YD+OYR/QuCHNpnlbG7ahvb8fO15Ei6uqCzO3PVB3bZdoMV02fLyCNlP1/VEjWtgtadutXpvkP2gTLk9ajW8868VEuqietpghmbxzzUjz4CHZvRi4H23Z4y9oxVl6bIh6dWDXqup278dQ9v65wGTvyiG0Xju9NZzcvqouGz6/Hq3paHc37Fpd6aRJblqD5jcCZ9LeyIH+Zq6HfWh791QBblYNBTqewcbJeTdFuACSnEF7j//ZcHxN4As97VEb7kMeXFVdJhMASf6uqv5p7DiWI8k9pg4vB75VVReNFc9ikrwXeAnwStqM1VOBtb0VNUryc9pr+0uAj/daQCjJ+bS9s5+YDCitVmGwWU/U/gX4AvDv1ek/pPdkctpQTn5f2jKErye5IXDrqvrYyKHNkeSBtH1gO1TVHkn+CDiywxH206rq9knOrKpbT58bO7ZpST5Fm1Xbjjaz9n1aFThn1a6EJI+oqrdP7bOZo6P9NcvSy2tXki9WZ32pFpLko8CBVfW/Y8cy64bk/LW016P1swAL7Pke1VBU5A41tFwZqiueOnm970WSdVW1duw4FpPkFWy6AFcX2xqGYmYf77mY2USSnWn7je9LW9H1MeCpvSVCSQ6gVcbdB/g1bcXHSVV14qiBzZPWM3Ef4OW0ir6PAD65GonazC59HDwBeDpweZJJI8+uNp4CJyb5C/pOJqf30X1q6tyvaEvjevM82hPmUwBV9eW0fja9+dUwmvn1JIfRZiuvNXJMC7lOVf10KJP7tqp67jBSrCvnmsPHa2/yqtnRy+vVy4bSyB9jbmGJL40X0oJ+Dnw5rZH4dJxd3GTOmMur6jVjB7EMb6ZVenw/7f7jANrsb28+nuQZwLuZu6+7lz2pk/uMu9KWEr57OD6QtpS4CzNSzAyAqvoBrbpn16rqg8AHk9ySVvL+r4H/C1x91MA2lqq6HHhikscAnwWutxrfeKYTtarq9oYoyT9V1d8xG8nk9D66XYEfDZ9fl7ahu7eNspdV1U/mrb2+YrGLR/RUWp+/pwDPp1WtW6h31di2G2ZPH0Jr0K2roKpeN4y4/rSqXjp2PFuQWwOPpD1/Js/zYm4VyB4cN/zRb+9DSZ4IvJ+5SW8viQXQZsmHFQl3o/1OHtJTleQpDx0+Tu+pKToph15VbwVI8lfA3YYb4kmv3C6qUya5U1WdTOfFzGZldnIiyftoZe+/QdvP+yjgi6MGtbDXTj6pqrcMs+mrskdtJhO1JJucauxkpHVf4O96TiYnauijluT1wPur6vjh+P7An40Z2yLOSvIwYNuhYMdTaNPlXamqU4dP/5e2P61XRwIn0Jo3njrMTn595Jhm0jDiejBtX+Ksy9KXrIoDgZtW6w/UrcnNplbEZEDrmVPnukks5vkNLbaizwHDmemVSpuh2IkN1WevxSrNWizDq2l793svZtbjKqhNeQGtN1mPhU7Wq3kNuIdl2KvSMH4m96gl+eTw6Y7AWuArtJuKvYF1HW2AvyeL3Oz0NjIIbb39/LX1C50b27CX7tnAn9D+f0+g9Xv75Sb/4iobNsQ/E9iNjjfEa2WlNT7eno2XGfUwgLRsSW5VVV/tII4PAIdWVY+VwNabgSqvWmFTVR/fR3sv+nNaxeQuqj4ut2VIL5IcQtva8Ena/+cfA8/rYRBktQpHbI2GlhHzXze7aCDeg5lM1CaS/Dvw3Ko6czi+Fe1J/eBxI4Mkv6LtSVooUase37yTnEBbZjDptv5w4I+r6v8bL6rZNUMb4t/MAkslqmpVRou2NFMDSZP/08ly5y4S9CS70Hok3hj4KPCiqaqfH6iqrmbRh6VlewOnMncZXG/Fg2aiymvPkty7qj6xWILRYWLRddXH4bUdWs+0uwCfGI7vRSsY1V3j7iQ3YEPv1i9W1ffGjGciyY/ZRKuNDl+P1gB/y8YJUBfvQxPD/uN70uI8nrZP7bM93Mf3YiaXPk65xSRJA6iqryb5gzEDmnJ2DxXTrqSDaTca7x+OTxrOdWVWSjczOxviPzz1+Y60UeHvjBTLzJqq9vhhNuz5nOhpROxNtBmAk2k9HT+d5IFDNbDdRo1sYc8dO4BlunpVnZgk1RoiPy/JaYCJ2vLdg5ZMLNSTsKdehBNhbm+q39DPkuH1zeKTfAzYq6q+OxzfEHjLiKHNscB2lguHjzdKcqNOViNcwty+k717B21Vx37AX9KWE18yakQLezBtj9rpVXVIkuuzYbJAzH6idkaSNzB3BshqdVfRsBzzqWPHsQzvoc1UvYEOGzhOVdGclQ3x75s+TvIuWkUjXTmT/ai3AO4AfJB20/ZA4JSxglrAmqqabIx+cpJHACcl2Z++Ekqgv35pmzArVV67VVWTpPxxve9ZGUxXfYS2p7vHqo+7TJK0wX/TCof1YlMJUC+Fgy6dodcigN+tqjcmeeoQ96eTnLrk31p9v6yqK5JcnmQnWnugnn43RzfridohwF+xIbk4CehlBuNlYwdwZQ1T5f8X+EM6niqn/5mq6SqaMBsb4qftSVsqoyuhNjTBPAm4XVVdOhw/D/jIiKHNt32SHSd7Oqv1fvseba/nNTf9V1dPks9W1d2SXMrcBLLHyrmwcZXXe9FnlddZ8M0k/0GbEfhEr61thqqPn6aVlYd+qz6eOGxteNdw/FDg4yPGM8cs9CUDLhg7gCvpsuHjd5PsR1sl8zubuH4spya5LvB62r3T/wKfGzekvsz0HrVZMEsFJYblEe+mLStcP1VeVX87amCDqZmqp9BGXbqeqZoVC9wIfw941vyZNi1PknOBvavqV8Px1YAzquoW40bWJHka8KX5o8NJbgv8c1Xdb5zIZtfQluGFVfWMsWPZEgwFox4AHESrtPdh4Jiq6m6mf/jZX5+57+/fHi+ihQ37/u4+HJ5UVe/f1PVjmYXCEknuwsZbL3qL8QG0mgO7AK+gVdP8h6rqqoVIkk8A/1JVxyfZHbgO8KSqOnTUwDoy04lakrvSKgTNT4K6mbGYlYISAElOq6rbJzljshk6yalVdYexYwNI8k023vsz0V2BlqG4wKdpL5afm8ywaMuW5Nm0nnTTy6HeXVUvGC+qjSW5a1V9bqlzWp4kJ1fVncaOY0uT5Hq0FSoPr6ptx45nWpIn0/ZQ/jcb9qdVL8VEZs0sFJZI8m/AzYAvs+Gerqqz/mSzIsn5tD2Jn5halWKFzSmznqh9DXgaGydBPxwtqHkmyc/YcSzH5EZjWCLxctpU+Xur6mYjhzaTkuxBG8G8O3An2uzfZ6rqaaMGNk+SE6vqPkud0/INm+OnR6+7Ww610Juhb5BXXZLX0Cppvoe5bRl6K4AxE5Lcg7ZEb19ab6h39zbLn+Q84I493XMsZJhNeyFtSXvodPlwWhPhSWGJ20wKS/Q0y5/kHFphlq5vnoetLI9n45m/rqo5J/kSsA/tnnMX4BHAJ30f2mDW96j9pKo+OnYQS5iJghKDf0xyHeBv2DBV3lVSAZDkQOA/qurSJM+hLY15fm83w1X1zSS/BH49/LkX0EtVUpLsSNtTs/Mwaj2ZqdyJdsOpq2ioUtZDpbKNJLkzrVT3mqlKldB+7l3NWMyYHYEfMrfwQY+VCruX5ALgdOBY4JmT8vcduhD4ydhBLMM/Aw+sqnPGDmQJv1igsMQuYwc1z1eBGwDfXerCkX2Qtprn43RYdG1Kqupy4IlJHkMrZNZLk/MuzHqi9skkL6K9EU4nQT3dIE02k3dfUKKqJmXaf0JLKnr191X1niR3A+5L6wn1Wjb0XulCkm8APwDeSasE9uSqumLcqOZ4AvDXwI2Ym1T8FHjlKBFpNexAq0a4HRsqVUL7uXezxGjWTEqha0XsXVU/HTuIxUwNcJwPfCrJR5h7D/KSUQJb3H/PQJIGsG6BwhJfGDekjewMnJ3kFDru6whco5f6AkuYVCCmqt4yzKo+acR4ujPrSx8/ucDp6rFQxyyYoany06vqtkleAJxZVe+cnBs7tmlJngrcjTYi+DXafrWTquobowY2T5InV9Urxo5DqyvJblX1rWHkutxD+dsZljo/mY1fP3u7geveUITrNcD1q+pWSfYG9q+qfxw5NGD9XqpFTfba9CLJy2izQB9gbnLR7WzvUFhip6rqquXSsCR3I72V7k/yj7Sm5sePHYt+OzOdqPUsyb2r6hPD2vCN9PgCmeTztKny+Xv+etsX8GFaj6L70ZY9/gI4papuM2pgi0hyLVoriWcAN+lwQ/wOtCqffzyc+hTwuqq6bNG/pJmXZC2tD9RkVu0nwP/psdDRLBgKR70ROBNYP3Pe2w3cLBhK3j+T9jp02+HcV6vqVuNGNpuSvHmB09XbICxAkhuzcYG4k8aLaGNJdgP2rKqPDxVKt+1loGteFedr0RLzy4fj7vYlamkzvfRx2Gj6T8CNqur+SfYC7lxVPTScvAfwCVqz2/l63bcwK1PlD6FtMP+Xqvpxkhsyd2lpF5K8mFZQ4prA54EjaIlwb14NbD98BHgkbTT7caNFpNXwJuCJVfUZgGEp8ZsBK9ZdNb+sqpePHcQW4hpVdUoyp8Dv5YtdPJZh5u8ZbDyL2tWqnllZlpvkhbQCMmczVVGR1iO3C0keDxxK60l2M9p+7tcCXRTfqqprAyR5O+3/7TMzsuxVi5jpGbUkH6XdWDx7qBC0Ha1a0K1HDm29JHtU1TeXOteD3qfKp/qoLai3Ai1JHkxr3LgrcLXJ+Q5HB78yfzZyoXPasiy0XNiqj1ddkofRmsV/jH73TM+E4b39MOA9VXW74bX0sVV1/5FDm2NW2u/0vpR0IvN6UPYoyZdpVQq/ODXbe2ZP950ASe7FhqrTN6PtQ/9MVb1s1MB0pc30jBqwc1Udm+RZAFV1eZLeqtu8j7Y8b9p7gW5K9k9NlQf4uyS/onW1762E72lsiHNX4EfD59cFvg3sMV5oC7oe7abtJrSeK3eibYzuarQV+E2Sm032ziW5KX1XidLK+HSS1wHvoj2vHkorjHA7MMG4Cm5Nm42+NxuWPhb9Pd9nwXOB1wG3THIx8E3g3eOGtKDLq+o1YwexDK9nWEoKUFVnJHkn0FWiRivOsj1TAx0d+lVV/Xoy2ztMEHQ341FVn0xyEnAHWnG4vwRuRetJqBky64naz5L8LsOTJMmd6KRUbpJbAn8IXGfePrWdaGWcuzGZKof1s1Z70lmMAFW1B0CS1wPvn8z8Jbk/ralwb55Ce5H8/9u792C7q/KM498HmgGBELTFC9WKRi5FjIIiCGrxhhYDFoR6wSKDdhQqRSyI11KF0kFK7ZSOWIpS5FLqpSJEDaKghSANEiFYGqwScBCdSgWkEcLt6R9r7WTnZOeEpMlZv98+z2cmc85ee++Tl8PJPvv9rXe973W2X1F/Jk5tHNMoJ1A6qN5Wb29POVMX422wY/rn9eOgzmw3kmCsj0OBZ9t+qHUgY+DTlI7JtwGbAHMpHWq71vSoL+N3elFKCvwauFHSt1j1+9mlYdLfkfQh4AmSXgMcDVzWOKbV1O/hlpSLw1cDe9j+77ZRxfroe6L2PuBSYLakBcC2dKe99E6UXy7bsOo5tfspnRU7R9I7gWNZdQfoWjpSez1kL9srvoe2vy7pEy0DWoMHbT8om4yfQQAADi1JREFUCUmb2V4iaafWQY2wgHKl9VXAvcDldK8lcmx481i5Q039/D7gBts3Nouqv35Aeb3Pm6H/v0Mog8PfSindOhzYr2lEo/Vl/M7dkmaz8qL2IXRzDtil9U+XfQB4B6Vp0LuArwHnNI1otMWUyq1dKa/r90r6ru0H2oYV66q3Z9QkbUrZsTiTkhQJuLVrneokvcR2L9701vkVgx2gFwx2gGyP7FzZiqTLKVeILqhLhwEvt/3adlGtTtKXKTtT76XsTtwDzLC9f9PAJpD0ecoMrQvr0luBbWwf2i6q2Nhq6dOLKG+MRLmwtJjSce2Ltrt48aOzJH2b0ojlero9X6kX6rmqSyhl7QflDeb6q+XsZ1MG3d9DKSU9zPYdTQMboXYh3rHe7Nx7ur6RNBM4gtL05qm2N5v8GdE1vU3UACQttP3i1nFMpu70nEJpIT+f8ov8ONsXTPrEBiRdb3uPelh2T9vLJf2H7ee2jm1YLc88iZXt5P8N+FgHy01WqLNXZgHzu1YaJekW27usbS3GSz2/sL/t/623twK+SumoekP+/6+bvsxX6rJ6sXD4TcmTKbsBywFsd6IjaZ/G79SL2qfZPl7SlsAmXWklP5GkfYHzgNspF4+eAby9Cw24RvxsrqIrP5sDkt5D2Y1+IeX7eTWlmciVLeOKddf30scFkv6ecsh42WCxY4fg97P9fkkHUf6xHExJLDqXqAF3StqGchXzCkn3AJ274lYTsmNbx7EuOv5mbZGkvWxfByBpT+B7jWOKje/JrHpo/2FKV7gHakOhWAcd/zfeF3NbB/A49Wb8ju1H6+gNbC9b2+MbO4PynulWWLGr+s90o/na4GfzT+rH8+vHt9HBZiKUPgN/Q7no1sXziPE49X1H7aoRy+7SDJPBjpSkcyjlRPP70Pq84ztA2wLvpzRrWdH0pEv/3/tg6ArhDEr58E/q7WcCS7KjMt4kfRQ4CPhKXTqAUgZ5BnC27cNaxdYnkq6x/dIJg2ahe11zYwOTtKntznfIlXQWZd7XF1j1onZnEkoASYsn7kyNWmspY01iqvV6R832K1rH8DhcJmkJpfTxqJpkPNg4prXq+NXhCym7qHMpLWffDvyiaUT91Jer17ER2D65zqvapy692/ZgJzVJ2uNke7BbMXNtj42xs1TSfMrvoyvd3SvfmwP/w6qdXDu181d9r17UHj5/3rXqDknax/aCemNvSmfSiI2i1ztqAJJez+o7Kx9vF9Hq6pmq+2oJwhbA1rZ/3jquvpJ0g+0XDl9pG5yvax1bRERMD/X3+VzgzZR5qfOAi21f0zSwnpK0GaW08KV16WrgU10agC3phcBnKRVHojRnObJjR25ijPQ6UZP0aWALyjC/cygtfRfafkfTwIZIOnzUuu3PTXUs40LSdbb3qt0f/w64i1JWOrtxaBERMQ1JeiJlmPBhtjdtHc8wSZtTWspPvKh9ZLOg1qBWHWG701UykmYB2O7E7N4YX70ufQT2tj2n7qx8TNIZwNdbBzXB8C7P5pRZVYuAJGrr75T6IvlnlPEMWwPHtQ0pIiKmm3qe+02UbqnfA/6wbUQjnQ8sAV4LfJxSUvifTSMaojKJ+yTgPdQyQkmPAmd2rUIKVq3kGgwR72KcMR76nqgN5qr8WtJ2lBrspzWMZzW2jxm+XbsqXtwonLFge1799D7KbmpERMSUknQ78H3g88AJHe6q+Bzbh0p6g+3z6gzFq1sHNeQ4ylnZPWwvhRWz386SdJztTzaNbsiaKrmaBhVjre+J2rya+JxO2aUy3ZwQP2wZ8KzWQfRZLY34Y2B7hn6Gu1jGERERY2uO7V+1DuJxGAyNvlfSrsDPKeM5uuKPgNfYvnuwYPs2SW8DvgF0JlGjH5VcMUZ6najZPrl++iVJ84DNu1YvLOkyVrZs3hTYhXL1LdbfVyhXA78JdL41ckREjKWnSvoyZf7grpLmAAfaPqV1YBOcXc/QfYQygmMr4KNtQ1rFjOEkbcD2LyTNaBHQJDpfyRXjpZeJmqSXT3IfXZhiP+SvWZmoPQLcYfunDeMZB1vYPrF1EBERMa39I3AC8A8AthfXssKuJWrnA2+kVKGcV9ee0iya1U02q7VTc1zpZyVX9FgvEzXKC+NEBuYAz6DsXDU1NPxUE+6ypOXAj4EP2/7WlAfXf/Mk7W/7a60DiYiIaWsL2wsHDSWqR1oFM4mvUM503wB0ptX9kOdLGlVCKoa6VHZBHyq5Yrz0MlGzfcDwbUn7ULb0fw4cM/JJU2yy4aeSNgV2pQxu3nXKguq5Ccnvh2rC+3C9bdtbt4wvIiKmlbslzaZWzUg6BPhZ25BGerrt17UOYk26Ns5gFEkHT3LfcuDHtpdMYUgxTfQyURuQ9CpKnbWBU21f0Tikx8X2o8BNks5sHUufDCe/dYj4DnTsaltEREwbJ1HKHneW9FNgKfAvbUMa6VpJz7N9c+tARqm/z9fI9i+nKpZJHDDJfb8B/K6ka23/6VQFFNNDLwde1xkWH6Zs5f+l7WsahxRTSNI7gWOBpwM3AnsB19p+VdPAIiJi2pC0CHg7cBtl/tdc4L2292wa2ASSbgGeQ0kkl7OyCmVO08AqSUsZfVQESpzPnuKQ1pmkTYCbbT+3dSwxXvqaqD0G3AncxMpGHSvYPnDKg4opI+lmyiDx62y/QNLOlB3VNZYmREREbEh11tcXgLcCLwMOBw7o2pklSc8ctW77jqmOpe8kPQU4FdjO9u9L2gV4ie3PSHqa7S6WvkaP9bX0MUOOp7cHbT8oCUmb2V4iaafWQUVExPRRZ329BbgE+AnwWtsPrOVpU65PCVkdI7DKsYaOdfL+J+BcSlUXwA8p5a6fSZIWG0MvEzXb32kdQzR1Z22PewlwhaR7gN78IoqIiP6qVR3D1TxPonSb/vc6IqgTJYV9s4ZjDd8FXtkyrgl+y/bnJX0QwPYjkjLPNTaaXiZqI14kV9wFPGb7+VMcUkwh2wfVT/9C0lXALGB+w5AiImL6mNs6gDF1LCuPNbxicKyhcUwTLZP0m6zs9LkXpV9CxEbRy0SN0S+SosxQ++AUxxINZXc1IiKmUp9KCXumD8ca3gdcCsyWtADYFjikbUgxznqZqA2/SErajXKQ91BKR6MvtYorIiIiItZL54812F4k6feAnSgbBLfafrhxWDHG+tr1cUfgLfXP3ZSDnMfbHtnZKCIiIiL6oSZDs4D5th9qHc8wSXsD2zO02WH7c80CirHW10TtMeBq4B22f1TXbuvDrI2IiIiIWJ2klwI72D5X0rbAVraXto5rQNL5wGxKs5NBExFn0HVsLL0sfQQOBt4MXCVpPnAxowclRkRERETHSToJeBGlrPBcYAZwAbBPy7gmeBGwi/u4yxG9tEnrANaH7UtsvxnYGbgKeC/wZElnSdqvbXQRERERsY4OAg4ElgHYvguY2TSi1f0AeGrrIGL66OuOGgC2lwEXARfVIYmHAicC32gaWERERESsi4dsW9Kg9f2WrQMakHQZpSX/TOAWSQuB5YP7bR/YKrYYb708oxYRERER40GSgI8Cvw28Bvgr4EjgIttntowNVjQ3EXAa8P7hu4DTbO/ZJLAYe0nUIiIiIqIpSTdT5pTtR0mALrd9RduoViVpke3dJ6wttj2nVUwx3npd+hgRERERY2ERcK/tE1oHMpGko4CjgWdLWjx010xgQZuoYjrIjlpERERENCVpCfAcypDrZYP1LuxWSZoFPJFSkvmBobvut/3LNlHFdJBELSIiIiKakvTMUeu275jqWCK6IolaREREREREx/RyjlpERERERMQ4S6IWERERERHRMUnUIiKiNySlW3FEREwLSdQiIqIJSVtK+qqkmyT9QNKbJO0h6dq6tlDSTElHSLpU0pXAt+rzPlvv/76kN9Svt6mk0yVdL2mxpHfV9X0lfVvSFyUtkXRhHbC7prhul/QxSYsk3Sxp57r+YknfrX/ntZJ2qutHSLpE0hX1ue+R9L76uOskPak+brak+ZJukHT14OtGRESMkiuTERHRyuuAu2y/Hla0wP4+8Cbb10vaGnigPnZ3YI7tX0o6FbjS9pGStgEWSvomcBhwn+09JG0GLJD0jfr83YDnAndR5h7tA1wzSWx3295d0tHA8cA7gSXAy2w/IunVwKnAG+vjd61/x+bAj4ATbe8m6ZPA4cDfAmcD77b9X5L2BD4FvHK9v3sRETHWkqhFREQrNwNnSDoNmAfcC/zM9vUAtn8FUDe/rhiaV7QfcKCk4+vtzYHfqetzJB1S12cBOwAPAQtt31m/3o3A9kyeqP1r/XgDcPDQ1ztP0g6AgRlDj7/K9v3A/ZLuAy4b+m+cI2krYG/gC0ObeZtN+t2JiIhpLYlaREQ0YfuHknYH9gdOAa6c5OHLhj4X8Ebbtw4/oJYzHmP78gnr+wLLh5YeZe2//waPH37syZSE7CBJ2wPfHvF4gMeGbj9Wn78JcK/tF6zl742IiAByRi0iIhqRtB3wa9sXAKcDewJPk7RHvX/mGpqHXA4cMzhnJmm3ofWjJM2o6ztK2nIDhjwL+Gn9/Ih1eWLdHVwq6dAamyQ9fwPGFhERYyY7ahER0crzgNMlPQY8DBxF2S07U9ITKOfTXj3ieSdTznwtlrQJsBSYC5xDKWlcVJO4XwB/sAHj/QSl9PEjwFfX4/mHAWfV588ALgZu2oDxRUTEGJHt1jFERERERETEkJQ+RkREREREdExKHyMiYlqS9GXgWROWT5zYjCQiIqKFlD5GRERERER0TEofIyIiIiIiOiaJWkRERERERMckUYuIiIiIiOiYJGoREREREREdk0QtIiIiIiKiY5KoRUREREREdMz/AZXEjR85zDACAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": [],
      "needs_background": "light"
     }
    }
   ]
  }
 ]
}