classification_baseline_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zkpQmum8Zz-F"
   },
   "source": [
    "# XGBoost optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dnccsgbS3Dj6"
   },
   "source": [
    "This script implements the feature-based classifier proposed in\n",
    "\n",
    "Spinde, T.; Rudnitckaia, L.; Mitrović, J.; Hamborg, F.; Granitzer, M.; Gipp, B. & Donnay, K.\n",
    "Automated identification of bias inducing words in news articles using linguistic and context-oriented features \n",
    "Information Processing & Management, Elsevier, 2021, 58, 102505\n",
    "\n",
    "It will serve as baseline model to compare our system with it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zAd2yL3gZz-O"
   },
   "source": [
    "To run this script, you need the following files found in the /data directory:\n",
    "- \"final_labels_SG1.xlsx\"\n",
    "- \"final_labels_SG2.xlsx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6556,
     "status": "ok",
     "timestamp": 1621011892646,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "lLTGBVBsZz-P",
    "outputId": "e4d29228-16f7-4dbb-cc74-831a73498607"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting scikit-plot\n",
      "  Downloading https://files.pythonhosted.org/packages/7c/47/32520e259340c140a4ad27c1b97050dd3254fdc517b1d59974d47037510e/scikit_plot-0.3.7-py3-none-any.whl\n",
      "Requirement already satisfied: joblib>=0.10 in /usr/local/lib/python3.7/dist-packages (from scikit-plot) (1.0.1)\n",
      "Requirement already satisfied: matplotlib>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-plot) (3.2.2)\n",
      "Requirement already satisfied: scipy>=0.9 in /usr/local/lib/python3.7/dist-packages (from scikit-plot) (1.4.1)\n",
      "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.7/dist-packages (from scikit-plot) (0.22.2.post1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.4.0->scikit-plot) (1.3.1)\n",
      "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.4.0->scikit-plot) (1.19.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.4.0->scikit-plot) (0.10.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.4.0->scikit-plot) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.4.0->scikit-plot) (2.8.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib>=1.4.0->scikit-plot) (1.15.0)\n",
      "Installing collected packages: scikit-plot\n",
      "Successfully installed scikit-plot-0.3.7\n"
     ]
    }
   ],
   "source": [
    "# misc\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import ast\n",
    "import warnings\n",
    "import math\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "from xgboost import plot_importance\n",
    "import seaborn as sns\n",
    "\n",
    "# data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "\n",
    "# ML\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import metrics\n",
    "\n",
    "!pip install scikit-plot\n",
    "import scikitplot as skplt\n",
    "import xgboost "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BljLo9ejZz-R"
   },
   "source": [
    "## 1 Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "executionInfo": {
     "elapsed": 184235,
     "status": "ok",
     "timestamp": 1621016162824,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "JjYkYmFRHTwr"
   },
   "outputs": [],
   "source": [
    "data = pd.read_excel('data/dt_final_SG2.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "executionInfo": {
     "elapsed": 183441,
     "status": "ok",
     "timestamp": 1621016163146,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "7xijAlvvZz-R"
   },
   "outputs": [],
   "source": [
    "# only features and label\n",
    "dt = data[[\n",
    " 'sentence',\n",
    " 'label4', \n",
    " 'glove_vec300_norm', \n",
    " 'tfidf_art', \n",
    " 'negative_conc',\n",
    " 'positive_conc',\n",
    " 'weak_subj',\n",
    " 'strong_subj',\n",
    " 'hyperbolic_terms',\n",
    " 'attitude_markers',\n",
    " 'kill_verbs',\n",
    " 'bias_lexicon',\n",
    " 'assertive_verbs',\n",
    " 'factive_verbs',\n",
    " 'report_verbs',\n",
    " 'implicative_verbs',\n",
    " 'hedges',\n",
    " 'boosters',\n",
    " 'affect ',\n",
    " 'posemo ',\n",
    " 'negemo ',\n",
    " 'anx ',\n",
    " 'anger ',\n",
    " 'sad ',\n",
    " 'social ',\n",
    " 'family ',\n",
    " 'friend ',\n",
    " 'female ',\n",
    " 'male ',\n",
    " 'cogproc ',\n",
    " 'insight ',\n",
    " 'cause ',\n",
    " 'discrep ',\n",
    " 'tentat ',\n",
    " 'certain ',\n",
    " 'differ ',\n",
    " 'percept ',\n",
    " 'see ',\n",
    " 'hear ',\n",
    " 'feel ',\n",
    " 'bio ',\n",
    " 'body ',\n",
    " 'health ',\n",
    " 'sexual ',\n",
    " 'ingest ',\n",
    " 'drives ',\n",
    " 'affiliation ',\n",
    " 'achieve ',\n",
    " 'power ',\n",
    " 'reward ',\n",
    " 'risk ',\n",
    " 'focuspast ',\n",
    " 'focuspresent ',\n",
    " 'focusfuture ',\n",
    " 'relativ ',\n",
    " 'motion ',\n",
    " 'space ',\n",
    " 'time ',\n",
    " 'work ',\n",
    " 'leisure ',\n",
    " 'home ',\n",
    " 'money ',\n",
    " 'relig ',\n",
    " 'death ',\n",
    " 'informal ',\n",
    " 'swear ',\n",
    " 'netspeak ',\n",
    " 'assent ',\n",
    " 'nonflu ',\n",
    " 'filler ',\n",
    " 'pos_ADJ',\n",
    " 'pos_ADP',\n",
    " 'pos_ADV',\n",
    " 'pos_AUX',\n",
    " 'pos_DET',\n",
    " 'pos_INTJ',\n",
    " 'pos_NOUN',\n",
    " 'pos_PRON',\n",
    " 'pos_PROPN',\n",
    " 'pos_SCONJ',\n",
    " 'pos_VERB',\n",
    " 'pos_X',\n",
    " 'dep_ROOT',\n",
    " 'dep_acl',\n",
    " 'dep_acomp',\n",
    " 'dep_advcl',\n",
    " 'dep_advmod',\n",
    " 'dep_agent',\n",
    " 'dep_amod',\n",
    " 'dep_appos',\n",
    " 'dep_attr',\n",
    " 'dep_aux',\n",
    " 'dep_auxpass',\n",
    " #'dep_case',\n",
    " 'dep_cc',\n",
    " 'dep_ccomp',\n",
    " 'dep_compound',\n",
    " 'dep_conj',\n",
    " 'dep_csubj',\n",
    " 'dep_dative',\n",
    " 'dep_dep',\n",
    " 'dep_det',\n",
    " 'dep_dobj',\n",
    " 'dep_expl',\n",
    " 'dep_intj',\n",
    " 'dep_mark',\n",
    " #'dep_meta',\n",
    " 'dep_neg',\n",
    " 'dep_nmod',\n",
    " 'dep_npadvmod',\n",
    " 'dep_nsubj',\n",
    " 'dep_nsubjpass',\n",
    " 'dep_nummod',\n",
    " 'dep_oprd',\n",
    " 'dep_parataxis',\n",
    " 'dep_pcomp',\n",
    " 'dep_pobj',\n",
    " 'dep_poss',\n",
    " 'dep_preconj',\n",
    " 'dep_predet',\n",
    " 'dep_prep',\n",
    " 'dep_prt',\n",
    " 'dep_punct',\n",
    " 'dep_quantmod',\n",
    " 'dep_relcl',\n",
    " 'dep_xcomp',\n",
    " 'ne_CARDINAL',\n",
    " 'ne_DATE',\n",
    " 'ne_EVENT',\n",
    " 'ne_FAC',\n",
    " 'ne_GPE',\n",
    " 'ne_LANGUAGE',\n",
    " 'ne_LAW',\n",
    " 'ne_LOC',\n",
    " 'ne_MONEY',\n",
    " 'ne_NORP',\n",
    " 'ne_ORDINAL',\n",
    " 'ne_ORG',\n",
    " 'ne_PERCENT',\n",
    " 'ne_PERSON',\n",
    " 'ne_PRODUCT',\n",
    " 'ne_QUANTITY',\n",
    " 'ne_TIME',\n",
    " 'ne_WORK_OF_ART',\n",
    " 'negative_conc_context',\n",
    " 'positive_conc_context',\n",
    " 'weak_subj_context',\n",
    " 'strong_subj_context',\n",
    " 'hyperbolic_terms_context',\n",
    " 'attitude_markers_context',\n",
    " 'kill_verbs_context',\n",
    " 'bias_lexicon_context',\n",
    " 'assertive_verbs_context',\n",
    " 'factive_verbs_context',\n",
    " 'report_verbs_context',\n",
    " 'implicative_verbs_context',\n",
    " 'hedges_context',\n",
    " 'boosters_context',\n",
    " 'affect _context',\n",
    " 'posemo _context',\n",
    " 'negemo _context',\n",
    " 'anx _context',\n",
    " 'anger _context',\n",
    " 'sad _context',\n",
    " 'social _context',\n",
    " 'family _context',\n",
    " 'friend _context',\n",
    " 'female _context',\n",
    " 'male _context',\n",
    " 'cogproc _context',\n",
    " 'insight _context',\n",
    " 'cause _context',\n",
    " 'discrep _context',\n",
    " 'tentat _context',\n",
    " 'certain _context',\n",
    " 'differ _context',\n",
    " 'percept _context',\n",
    " 'see _context',\n",
    " 'hear _context',\n",
    " 'feel _context',\n",
    " 'bio _context',\n",
    " 'body _context',\n",
    " 'health _context',\n",
    " 'sexual _context',\n",
    " 'ingest _context',\n",
    " 'drives _context',\n",
    " 'affiliation _context',\n",
    " 'achieve _context',\n",
    " 'power _context',\n",
    " 'reward _context',\n",
    " 'risk _context',\n",
    " 'focuspast _context',\n",
    " 'focuspresent _context',\n",
    " 'focusfuture _context',\n",
    " 'relativ _context',\n",
    " 'motion _context',\n",
    " 'space _context',\n",
    " 'time _context',\n",
    " 'work _context',\n",
    " 'leisure _context',\n",
    " 'home _context',\n",
    " 'money _context',\n",
    " 'relig _context',\n",
    " 'death _context',\n",
    " 'informal _context',\n",
    " 'swear _context',\n",
    " 'netspeak _context',\n",
    " 'assent _context',\n",
    " 'nonflu _context',\n",
    " 'filler _context',\n",
    " 'pos_ADJ_context',\n",
    " 'pos_ADP_context',\n",
    " 'pos_ADV_context',\n",
    " 'pos_AUX_context',\n",
    " 'pos_DET_context',\n",
    " 'pos_INTJ_context',\n",
    " 'pos_NOUN_context',\n",
    " 'pos_PRON_context',\n",
    " 'pos_PROPN_context',\n",
    " 'pos_SCONJ_context',\n",
    " 'pos_VERB_context',\n",
    " 'pos_X_context',\n",
    " 'dep_ROOT_context',\n",
    " 'dep_acl_context',\n",
    " 'dep_acomp_context',\n",
    " 'dep_advcl_context',\n",
    " 'dep_advmod_context',\n",
    " 'dep_agent_context',\n",
    " 'dep_amod_context',\n",
    " 'dep_appos_context',\n",
    " 'dep_attr_context',\n",
    " 'dep_aux_context',\n",
    " 'dep_auxpass_context',\n",
    " #'dep_case_context',\n",
    " 'dep_cc_context',\n",
    " 'dep_ccomp_context',\n",
    " 'dep_compound_context',\n",
    " 'dep_conj_context',\n",
    " 'dep_csubj_context',\n",
    " 'dep_dative_context',\n",
    " 'dep_dep_context',\n",
    " 'dep_det_context',\n",
    " 'dep_dobj_context',\n",
    " 'dep_expl_context',\n",
    " 'dep_intj_context',\n",
    " 'dep_mark_context',\n",
    " #'dep_meta_context',\n",
    " 'dep_neg_context',\n",
    " 'dep_nmod_context',\n",
    " 'dep_npadvmod_context',\n",
    " 'dep_nsubj_context',\n",
    " 'dep_nsubjpass_context',\n",
    " 'dep_nummod_context',\n",
    " 'dep_oprd_context',\n",
    " 'dep_parataxis_context',\n",
    " 'dep_pcomp_context',\n",
    " 'dep_pobj_context',\n",
    " 'dep_poss_context',\n",
    " 'dep_preconj_context',\n",
    " 'dep_predet_context',\n",
    " 'dep_prep_context',\n",
    " 'dep_prt_context',\n",
    " 'dep_punct_context',\n",
    " 'dep_quantmod_context',\n",
    " 'dep_relcl_context',\n",
    " 'dep_xcomp_context',\n",
    " 'ne_CARDINAL_context',\n",
    " 'ne_DATE_context',\n",
    " 'ne_EVENT_context',\n",
    " 'ne_FAC_context',\n",
    " 'ne_GPE_context',\n",
    " 'ne_LAW_context',\n",
    " 'ne_LOC_context',\n",
    " 'ne_MONEY_context',\n",
    " 'ne_NORP_context',\n",
    " 'ne_ORDINAL_context',\n",
    " 'ne_ORG_context',\n",
    " 'ne_PERCENT_context',\n",
    " 'ne_PERSON_context',\n",
    " 'ne_PRODUCT_context',\n",
    " 'ne_QUANTITY_context',\n",
    " 'ne_TIME_context',\n",
    " 'ne_WORK_OF_ART_context',\n",
    " 'ne_LANGUAGE_context']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 177496,
     "status": "ok",
     "timestamp": 1621016163401,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "VqX9cJUfZz-W",
    "outputId": "32739767-13b9-4b30-c436-07d58b034d37"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values:\n",
      "tfidf_art 1190\n"
     ]
    }
   ],
   "source": [
    "print('Number of missing values:')\n",
    "mis = (dt.isnull().sum()).to_frame()\n",
    "for i, row in mis.iterrows():\n",
    "    if row[0] > 0:\n",
    "        print(i, row[0])\n",
    "\n",
    "# Delete the rows with missing article text:\n",
    "dt = dt.dropna(subset=['tfidf_art'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 0
    },
    "executionInfo": {
     "elapsed": 175533,
     "status": "ok",
     "timestamp": 1621016163813,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "2I2wpV6ZbFGl",
    "outputId": "905d215a-2a26-43ba-8a34-aa8f937483e1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>label4</th>\n",
       "      <th>glove_vec300_norm</th>\n",
       "      <th>tfidf_art</th>\n",
       "      <th>negative_conc</th>\n",
       "      <th>positive_conc</th>\n",
       "      <th>weak_subj</th>\n",
       "      <th>strong_subj</th>\n",
       "      <th>hyperbolic_terms</th>\n",
       "      <th>attitude_markers</th>\n",
       "      <th>kill_verbs</th>\n",
       "      <th>bias_lexicon</th>\n",
       "      <th>assertive_verbs</th>\n",
       "      <th>factive_verbs</th>\n",
       "      <th>report_verbs</th>\n",
       "      <th>implicative_verbs</th>\n",
       "      <th>hedges</th>\n",
       "      <th>boosters</th>\n",
       "      <th>affect</th>\n",
       "      <th>posemo</th>\n",
       "      <th>negemo</th>\n",
       "      <th>anx</th>\n",
       "      <th>anger</th>\n",
       "      <th>sad</th>\n",
       "      <th>social</th>\n",
       "      <th>family</th>\n",
       "      <th>friend</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>cogproc</th>\n",
       "      <th>insight</th>\n",
       "      <th>cause</th>\n",
       "      <th>discrep</th>\n",
       "      <th>tentat</th>\n",
       "      <th>certain</th>\n",
       "      <th>differ</th>\n",
       "      <th>percept</th>\n",
       "      <th>see</th>\n",
       "      <th>hear</th>\n",
       "      <th>feel</th>\n",
       "      <th>...</th>\n",
       "      <th>dep_expl_context</th>\n",
       "      <th>dep_intj_context</th>\n",
       "      <th>dep_mark_context</th>\n",
       "      <th>dep_neg_context</th>\n",
       "      <th>dep_nmod_context</th>\n",
       "      <th>dep_npadvmod_context</th>\n",
       "      <th>dep_nsubj_context</th>\n",
       "      <th>dep_nsubjpass_context</th>\n",
       "      <th>dep_nummod_context</th>\n",
       "      <th>dep_oprd_context</th>\n",
       "      <th>dep_parataxis_context</th>\n",
       "      <th>dep_pcomp_context</th>\n",
       "      <th>dep_pobj_context</th>\n",
       "      <th>dep_poss_context</th>\n",
       "      <th>dep_preconj_context</th>\n",
       "      <th>dep_predet_context</th>\n",
       "      <th>dep_prep_context</th>\n",
       "      <th>dep_prt_context</th>\n",
       "      <th>dep_punct_context</th>\n",
       "      <th>dep_quantmod_context</th>\n",
       "      <th>dep_relcl_context</th>\n",
       "      <th>dep_xcomp_context</th>\n",
       "      <th>ne_CARDINAL_context</th>\n",
       "      <th>ne_DATE_context</th>\n",
       "      <th>ne_EVENT_context</th>\n",
       "      <th>ne_FAC_context</th>\n",
       "      <th>ne_GPE_context</th>\n",
       "      <th>ne_LAW_context</th>\n",
       "      <th>ne_LOC_context</th>\n",
       "      <th>ne_MONEY_context</th>\n",
       "      <th>ne_NORP_context</th>\n",
       "      <th>ne_ORDINAL_context</th>\n",
       "      <th>ne_ORG_context</th>\n",
       "      <th>ne_PERCENT_context</th>\n",
       "      <th>ne_PERSON_context</th>\n",
       "      <th>ne_PRODUCT_context</th>\n",
       "      <th>ne_QUANTITY_context</th>\n",
       "      <th>ne_TIME_context</th>\n",
       "      <th>ne_WORK_OF_ART_context</th>\n",
       "      <th>ne_LANGUAGE_context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Orange Is the New Black\" star Yael Stone is r...</td>\n",
       "      <td>0</td>\n",
       "      <td>7.128710</td>\n",
       "      <td>0.278468</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Orange Is the New Black\" star Yael Stone is r...</td>\n",
       "      <td>0</td>\n",
       "      <td>7.624207</td>\n",
       "      <td>0.126985</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"Orange Is the New Black\" star Yael Stone is r...</td>\n",
       "      <td>0</td>\n",
       "      <td>7.830023</td>\n",
       "      <td>0.134708</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Orange Is the New Black\" star Yael Stone is r...</td>\n",
       "      <td>0</td>\n",
       "      <td>7.506312</td>\n",
       "      <td>0.226964</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Orange Is the New Black\" star Yael Stone is r...</td>\n",
       "      <td>0</td>\n",
       "      <td>9.109019</td>\n",
       "      <td>0.303142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 280 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  ...  ne_LANGUAGE_context\n",
       "0  \"Orange Is the New Black\" star Yael Stone is r...  ...                    0\n",
       "1  \"Orange Is the New Black\" star Yael Stone is r...  ...                    0\n",
       "2  \"Orange Is the New Black\" star Yael Stone is r...  ...                    0\n",
       "3  \"Orange Is the New Black\" star Yael Stone is r...  ...                    0\n",
       "4  \"Orange Is the New Black\" star Yael Stone is r...  ...                    0\n",
       "\n",
       "[5 rows x 280 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 172267,
     "status": "ok",
     "timestamp": 1621016163814,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "_ut3SDFuc0fm",
    "outputId": "22fc74c1-5636-4a1b-db7f-85c9d12eb492"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3673"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dt['sentence'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "executionInfo": {
     "elapsed": 170532,
     "status": "ok",
     "timestamp": 1621016163815,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "w48XlZ8CZz-a"
   },
   "outputs": [],
   "source": [
    "x = dt.drop(['label4', 'sentence'], 1)\n",
    "y4 = dt[['label4']]\n",
    "sentence = dt[['sentence']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "executionInfo": {
     "elapsed": 1007,
     "status": "ok",
     "timestamp": 1621016165326,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "s6ApvLwDZz-b"
   },
   "outputs": [],
   "source": [
    "# train-test split\n",
    "train_features, test_features, train_labels, test_labels, train_sentences, test_sentences = train_test_split(x, y4, sentence,\n",
    "                                                                                                             test_size = 0.10, random_state = 42)\n",
    "train_features1, val_features, train_labels1, val_labels, train_sentences1, val_sentences = train_test_split(train_features, train_labels, train_sentences,\n",
    "                                                                                                             test_size = 0.10, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "executionInfo": {
     "elapsed": 994,
     "status": "ok",
     "timestamp": 1621016165328,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "e7pWACwBZz-b"
   },
   "outputs": [],
   "source": [
    "feature_names = list(x.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "executionInfo": {
     "elapsed": 1072,
     "status": "ok",
     "timestamp": 1621016166412,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "l2JCrBnAZz-b"
   },
   "outputs": [],
   "source": [
    "# matrix for XGBoost\n",
    "dtrain = xgboost.DMatrix(train_features, label=train_labels, feature_names=feature_names)\n",
    "dtest = xgboost.DMatrix(test_features, label=test_labels, feature_names=feature_names)\n",
    "dtrain1 = xgboost.DMatrix(train_features1, label=train_labels1, feature_names=feature_names)\n",
    "dval = xgboost.DMatrix(val_features, label=val_labels, feature_names=feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 802,
     "status": "ok",
     "timestamp": 1621016167236,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "sLfaNihMZz-c",
    "outputId": "35aa387e-833a-460b-a2e6-0b23743eaa6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features Shape: (58168, 278)\n",
      "Training Labels Shape: (58168, 1)\n",
      "Testing Features Shape: (6464, 278)\n",
      "Testing Labels Shape: (6464, 1)\n",
      "Training Features for final model Shape: (52351, 278)\n",
      "Training Labels for final model Shape: (52351, 1)\n",
      "Validation Features Shape: (5817, 278)\n",
      "Validation Labels Shape: (5817, 1)\n"
     ]
    }
   ],
   "source": [
    "print('Training Features Shape:', train_features.shape)\n",
    "print('Training Labels Shape:', train_labels.shape)\n",
    "print('Testing Features Shape:', test_features.shape)\n",
    "print('Testing Labels Shape:', test_labels.shape)\n",
    "print('Training Features for final model Shape:', train_features1.shape)\n",
    "print('Training Labels for final model Shape:', train_labels1.shape)\n",
    "print('Validation Features Shape:', val_features.shape)\n",
    "print('Validation Labels Shape:', val_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n50XEa1xZz-c"
   },
   "source": [
    "## 2 Baselines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h2AHIcWHZz-c"
   },
   "source": [
    "### B1: Random guesser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_EP2P2LIZz-d"
   },
   "outputs": [],
   "source": [
    "b1_pred = pd.Series(np.random.randint(2, size=len(test_features)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EFUT7u9_Zz-d",
    "outputId": "70336bba-22e9-4080-a842-887cd595c5c6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of b1, test:\n",
      "F1: 0.15\n",
      "Precision: 0.09\n",
      "Recall: 0.45\n",
      "AUC: 0.47\n",
      "Accuracy: 0.49\n",
      "Confusion matrix:\n",
      " [[1379  165]\n",
      " [1433  137]]\n"
     ]
    }
   ],
   "source": [
    "print('Performance of b1, test:')\n",
    "print('F1:', round(metrics.f1_score(test_labels,b1_pred),2))\n",
    "print('Precision:', round(metrics.precision_score(test_labels,b1_pred),2))\n",
    "print('Recall:', round(metrics.recall_score(test_labels,b1_pred),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(test_labels,b1_pred),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(test_labels,b1_pred),2))\n",
    "print('Confusion matrix:\\n', \n",
    "      metrics.confusion_matrix(test_labels,b1_pred).transpose())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3Y78Ta5AZz-d"
   },
   "source": [
    "### B2: Negative sentiment lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KWx9X6GfZz-d"
   },
   "outputs": [],
   "source": [
    "b2_pred = test_features.negative_conc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eiitDyRhZz-e",
    "outputId": "259a8f26-a8f1-42e9-92a4-2bcda83de3b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of b2, test:\n",
      "F1: 0.4\n",
      "Precision: 0.35\n",
      "Recall: 0.47\n",
      "AUC: 0.69\n",
      "Accuracy: 0.86\n",
      "Confusion matrix:\n",
      " [[2550  160]\n",
      " [ 262  142]]\n"
     ]
    }
   ],
   "source": [
    "print('Performance of b2, test:')\n",
    "print('F1:', round(metrics.f1_score(test_labels,b2_pred),2))\n",
    "print('Precision:', round(metrics.precision_score(test_labels,b2_pred),2))\n",
    "print('Recall:', round(metrics.recall_score(test_labels,b2_pred),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(test_labels,b2_pred),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(test_labels,b2_pred),2))\n",
    "print('Confusion matrix:\\n', metrics.confusion_matrix(test_labels,b2_pred).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "08lU2RCtZz-e",
    "outputId": "2ad366c1-b911-4e65-da37-ef2dc08979ee"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAggAAAGKCAYAAABpbLktAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3zM9x/A8ddlyhBJRAgiEjFqi1Ha2krV3ntUrRpFqdFSm1KUmEVtNaJm1U8RilB7NSJEYkSGEElkJ5f7/RE50su6XCIS72cf91Df7+fzufel6d37PlOhUqlUCCGEEEK8QS+vAxBCCCHEu0cSBCGEEEJokARBCCGEEBokQRBCCCGEBkkQhBBCCKFBEgQhhBBCaDDI6wCEEEKIgkCpVLJjxw727duHr68vSqUSe3t7Pv/8cwYPHoyxsbG67OXLl+nTp0+6bbVr145Fixaluubn58fy5cu5cuUKYWFhlClThh49etC7d2/09DS/7wcHB7Ny5Uo8PDwICQnBzs6O9u3bM2TIEIyMjDJ9PYr3bR+E9oq2eR2CEDrbHL0vr0MQIkdYmRjmSru6vtcfVP2hVXmlUsmIESM4deoUpqam1KhRAwMDA27cuEFERAQ1atRg8+bNmJiYALBlyxbmzp1LrVq1KF26tEZ7Li4u9O7dW/33O3fu0KdPHyIjI3FxcaFo0aJcuHCBiIiINJOJoKAgevToQVBQEJUrV8be3p6rV68SEhJCvXr12LBhA4aGGf/spQdBCCGE0JGbmxunTp2iYsWKrFu3juLFiwMQGhrKiBEjuHbtGqtWrWL8+PEAeHl5AfDtt99Su3btDNtWqVRMnDiRyMhIFi5cSIcOHdRtDxw4kEOHDvHpp5/SqlUrdZ0ZM2YQFBTEmDFjGDFiBADR0dGMHDmSc+fOsXXrVgYNGpTh88ocBCGEEAWOno7/aGvfvuReve+++06dHABYW1szY8YMAA4fPqy+fvv2bfT09Pjggw8ybdvDwwNvb2/q1aunTg5S2p4+fToAW7duVV/39fXl1KlTlClThuHDh6uvm5qaMnfuXPT19dm2bVumzysJghBCiAJHoVDo9NCWlZUVTk5OVK9eXeNe2bJlAXj69CkA8fHx3L9/HycnJ0xNTTNt+8yZMwC0aNFC417t2rUpWrQoV65cITIyEoCzZ8+iUqlo2rSpxtyEkiVLUrlyZZ48eYKPj0+GzysJghBCiALnbfcgrFmzhiNHjqT5gX/r1i0ASpQoAcC9e/dISEigVKlS/Pzzz7Ru3Zrq1avTrFkzFixYQERERKr6KR/kFSpUSPO5HR0dSUpK4v79+6nKly9fPs3yTk5OANy9ezfD1yQJghBCCJFLVCoVrq6uALRs2RJIHl4A+Pvvv9myZQv29vbUrl2biIgINmzYQLdu3QgNDVW3kdLzUKxYsTSfI+X6s2fPUpW3tbXNUvn0yCRFIYQQBY5eNoYJ3hQREaHxTR7AwsICCwuLLLezZMkSLl68iI2NDYMHDwZeT1CsV68ey5Ytw9raGkiedPjNN99w/vx5pk+fzvLlywGIiYkBoFChQmk+R8r16OjobJVPjyQIQgghChyFjh3kmzdvZsWKFRrXR40axejRo7PUxrJly1i7di1GRkYsXbpUnQhMmTKFfv36UaxYMczNzdXlra2tWbBgAZ999hnHjh3j6dOn2NraqucRpDc3ImW3gpQ/tS2fHkkQhBBCFDi69iAMGDCATp06aVzPSu9BYmIis2bNYteuXRgbG7N8+XLq1q2rvm9oaIijo2OadYsXL07lypW5fPkyt2/fxtbWVj2vITY2Ns06cXFxAOpyWS2fsidDeiRBEEIIIf5D26GEFFFRUYwZM4YzZ85gYWHBqlWrUiUHWWFjYwO8HiqwtbXFy8uLZ8+eUa5cOY3yISEhwOu5BSlzD9KbY5BSPr05CilkkqIQQogCR4GeTo/sCA8Pp1+/fpw5cwY7Ozu2b9+eZnIwZ84cRo4cyfPnz9Nsx9/fH3i96iFlNUJayxJVKhW+vr7o6+urk4eMygPq1Q7prYpIIQmCEEKIAkdPodDpoa34+HiGDh2Kp6cnzs7O7Ny5M90P4KtXr3L8+HHc3d017t29excvLy8sLS2pUqUKAA0bNgTgxIkTabYVGhpK7dq11fMZUsq7u7uTlJSUqnxAQABeXl6UKlUKZ2fnDF+TJAhCCCEKnLe9D4KrqyvXr1/Hzs6OrVu3qr/9p6VHjx4A/Pzzz+pv85C8imHKlCkolUoGDx6sPlCpXr16lC9fHg8PD3bv3p2q/MyZMwH44osv1Nft7e1p2LAhfn5+LFu2TH09OjqaqVOnolQqU5VPjxzWJEQ+JIc1iYIitw5r6m+U/kmJWbElfnuWy4aFhdG4cWNiY2OpUqWKeiOitCxatIikpCTGjh3L0aNHMTQ0pE6dOpiYmHDhwgWioqJo3bo1ixcvRl9fX13v5s2bDBgwgOjoaGrUqIGtrS0XL14kPDyc7t27M3v27FTP8/jxY3r16kVISAgVKlTA0dFRfVhTo0aNWL16NQYGGU9DlARBiHxIEgRRUBSEBOH06dMMGTIkS2W9vb2B5LkDu3btws3NDR8fH/T09HB2dqZ79+507do1zSWKPj4+uLq6cuHCBeLj43FwcKBnz55069YtVTKRIjAwEFdXV06fPs3Lly+xt7enQ4cODBgwINXR0+mRBEGIfEgSBFFQ5FaCMNCon071N8VvzbxQASfLHIUQQhQ4uu6DICRBEEIIUQDpupOikARBCCFEAaSnkARBV/ITFEIIIYQG6UEQQghR4CiQOQi6kgRBCCFEgSNDDLqTBEEIIUSBoyc9CDqTFEsIIYQQGqQHQQghRIEjyxx1JwmCEEKIAkc2StKdJAhCCCEKnOycyChSkwRBCCFEgZPWYUdCO5JiCSGEEEKD9CAIIYQocGSIQXeSIAghhChwZJKi7iRBEEIIUeDIMkfdSYIghBCiwJEeBN1JiiWEEEIIDdKDIIQQosCRIQbdSYIghBCiwJHTHHUnCYIQQogCR05z1J2kWEIIIYTQID0IQgghChyFDDHoTBIEIYQQBY4MMehOEgQhhBAFjkxS1J0kCEIIIQochfQg6ExSLCGEEEJokB4EIYQQBY+e9CDoShIEoaHX9N70mtFb63qDyw6i+cAW9JrRm4S4BMa6jOHx7Ufplrd1sGX9gw0AdDfvSmxUbLZjFiIj9WtWzXLZHr37Mm7iZADWrV7Jr7+sxtDQkM073HBydk63XsCTJ3Ru0woA93MXMTU11S1ooRs5i0FnkiAIDSGPQrh91lPjunOd8hgVMuLJ3SeEPw3TuB8fG6/+d0NjQ0av/5pJH3+LSqXK1XiFyKpyzuUxMzfPsEzJ0qU1riUkJDBv1nTWbtqKnp6MzOYHCulB0JkkCELD8Y3HOL7xmMb1dX6/Urxscdzm7cZ984lM26nUoBKfj2zD4RV/5EaYQmjtm0lTqF23Xrbq/nvzBnt27aB7rz45HJUQ7yZJhUWuSEpKAqDfvP7YlLbJ42iE0I3iVXf1muXLCA4KzONoRJYoFLo9hCQIIndcO3qV4AfBmBY25as1I/M6HCF08uFHH1PCriTR0dEsmDMrr8MRWaGn0O0hJEEQuSM2Ko7Vw1cCULdNXRr1bJTHEQmRfSYmJkya+gMA586e4a8jf+ZxRCJTkiDoTBIEkWuuHr3KqW0nARi8bCiFrQvncURCZF+Djz+h1edtAPh54Y+Eh2lO1BXvDoVCodNDSIIgctm6sesIDwnH0taSL5cMzutwhNDJuG8nY2llxYsXoSxbtDCvwxEiV0mCIHLVy+cR/DpuHQDNBjSnZouaeRyReJ+NHDKI+jWrpvv42z3j1TmWVlaMmTARgD//OMiF8+feRtgiO2SIQWeyzFHkulPbT9Gkb1NcPqvNiF9GMarqSOJj4vI6LPEeymwfhCKWlpm20bpNO44e/oN/znmwYM4sftuzj0ImJjkZpsgJMkygM0kQxFuxavhKlv+7khJOJeg7uy8bJvya1yGJ95Au+yC8aeL3P9Cna0cCnvjzy6oVjBn/bQ5EJ3KU9ALoTIYYxFvx9OFTtk/bBkC7Me1xrp3+lrVCvOtKlirF0BGjAdj92za8PP/N44iEBoWebg8hCYJ4ew4tO4j3BW/0DfQZtf5r9A308zokIbKtR5++VK5aDaVSybyZ01EqlXkdkhA5ShIE8daoVCpWDllOYkIiTjWd6PRt57wOSYhs09PTY8oPM9A3MODeXW+2b96Y1yGJNyj0FDo9hCQI4i17cOsB+37aC0DLIa3yOBohdFO+QkX69h8IwIG9e/I2GJGarGLQmSQI4q3bOWsH/t7+ciqeKBAGDfuKMg5l5dTSd42cxaAzeYcWb11CXAIrh65QH+gkRH5mbGzM5GnTZfc9UeBIgiDyhOfpfzm2/q+8DkOIHOFSpy7tO3XJ6zDEm2SIQWcK1XvWL9Ze0TavQxBCZ5uj9+V1CELkCCsTw1xpd94HS3Sq/53XNzkUSf4lGyUJIYQoeKQXQGeSIAghhCh4ZE6IzmQOghBCCCE0SA+CEEKIgkeGGHQmCYIQQoiCJw8SBKVSyY4dO9i3bx++vr4olUrs7e35/PPPGTx4MMbGxqnK37p1i5UrV3Lr1i2io6Nxdnamf//+tGvXLs32/fz8WL58OVeuXCEsLIwyZcrQo0cPevfunea+MsHBwaxcuRIPDw9CQkKws7Ojffv2DBkyBCMjo0xfj6xiECIfklUMoqDIrVUM82st16n+lGujtSqvVCoZMWIEp06dwtTUlBo1amBgYMCNGzeIiIigRo0abN68GZNXR4N7eHgwbNgwkpKSqFu3LiYmJpw/f57Y2FiGDx/OuHHjUrV/584d+vTpQ2RkJC4uLhQtWpQLFy4QERFBu3btWLRoUaryQUFB9OjRg6CgICpXroy9vT1Xr14lJCSEevXqsWHDBgwNM/7ZSw+CEEKIguct9yC4ublx6tQpKlasyLp16yhevDgAoaGhjBgxgmvXrrFq1SrGjx9PbGws336bfET4hg0bqF+/PgCPHj2iX79+rFmzhk8//ZSqVasCyefYTJw4kcjISBYuXEiHDh3UbQ8cOJBDhw7x6aef0qrV6+3rZ8yYQVBQEGPGjGHEiBEAREdHM3LkSM6dO8fWrVsZNGhQhq9JEoT3VKmKpekysQvVmlXH2s6auJg4Htzw46/1f3Fq20mN8lMP/kC9dvXSbe+Z/zMG2Q9Mda3jN50YtPjLDOMY5jyEwPuBqa7ZlLah98w+1PrMBYuiFoQGPOfCgQvsmr2Tl6Evs/4ihXglKSmJwf378MT/MUdPnU233J3bnmzdtIHrV68QHhZGURsbPvqkIV8OG4FNsWJp1vH18eHXX1Zz5fJFoqOiKGFXkk9bfUa/L76k0Ktvi6Lg27cvuVfvu+++UycHANbW1syYMYMOHTpw+PBhxo8fz4EDB3j+/DmdO3dWJwcAZcqUYfz48Xz77bds3bqVBQsWAMm9Dd7e3tSrV0+dHKS0PX36dHr37s3WrVvVCYKvry+nTp2iTJkyDB8+XF3e1NSUuXPn0qJFC7Zt2yYJgtBUt209Ju6ehLGJMXExcfjf8ceyuCVVG1ejauNquHxWmyV9U3dXOVRzAODO+TskpXGsbfjTcI1rKXUe3X5EZDof7HEx8an+butgy0//LMaqhBURzyN4cOsBpSuVpv3YDjTo3IBvG3xLaMDzbL1u8f76ZYUrt/+9RRFLy3TLHNz3OwvmzEKpVFLUxgYHR0ce+vmxb48bp0+dZO2mrZQqbZ+qzp3bnnz15UBiYmIoamODYzlnfH3u8evaNZw+dZI1G7dgZmaWy69OpOktL3O0srLCycmJ6tWra9wrW7YsAE+fPgXgzJkzADRv3lyjbLNmzdDX1+f06dPqaynlW7RooVG+du3aFC1alCtXrhAZGYm5uTlnz55FpVLRtGlTjbkJJUuWpHLlyty6dQsfHx+cnZ3TfU2SILxnLG0tGb99AsYmxhxd+z/WjV1HfEwcAB92qM+4Ld/QpE8T7l7w5o/lhwAwKWxC8bLFiY6IZuJHE7L8XGWrlwXg536LuX/1fpbqTHKbjFUJK/5af5Q1I1aTmJCIWREzJuz4ltqt6zBm4ximt/pBuxct3lsqlYpff1nN5g3rMyx357YnP86eiUqlYuyESXTv3Qc9PT1Cnj7luwnjuHXzBvNnz2TFL6/biYuL49uxXxMTE8OXw77iy2Ffoaenx9PgYCaOG82d27dxXfwTU36YkcuvUqTpLQ8xrFmzJt17t27dAqBEiRIA3Lt3D4AKFSpolDU3N8fW1pbAwECePXuGjY0NPj4+6ZYHcHR05Pnz59y/f58aNWqoy5cvXz7N8k5OTty6dYu7d+9mmCDIPgjvmU8Ht8TUwhSfKz6sGr5SnRwAXDjwD1umbAagw7jX3Vhlq5UF4PHtR1l+Hj09PUp/YE9SUhKPvfyzVKfmp7UoX7cCwQ+CWf3VKhITEgGICo9iYfcFRDwLp1ZLF5zrpP1LL8Sbnj97xqRxY1i/ZlWmZV2XLCIpKYl+XwyiZ99+6m9dxWxtmTHvRxQKBZcv/ENgQIC6zpFDBwl5Gky1GjUZ8tVIdR3b4sWZv2gpBgYG/HFwP89CQnLnBYoMKRQKnR45RaVS4erqCkDLli0BCHn1O1EsnWGrlOvPnj0DXvc8aFve1tY2S+XTIwnCe6Zak2oAnN97Ls3jaS/9cRGA4o4lMLNM7hp1eJUgPPLMeoJgV74kxibGBPsFp0pCMtKsfzMATv/2N8rE1MMYMZExnNmV3M32SfdPshyHeD9dOOdBtw5tOH3KnaI2Noz4emy6ZZ8GB3HtymVMzcwY8OVQjfulStszdsJEvpk0BQOD152uf/5xEIA27Tpo1LErWZJ69RugTEzk1InjOfCKxNsWERGBv7+/xiMiIkKrdpYsWcLFixexsbFh8ODBAMTExABQqFChNOukXI+Ojn4r5dPzzg0xREVFER0djUqlwtTUFHNz87wOqUDZPm0bp7ad5N5lnzTvFzJ7/Qulb6APvB4q0CZByE6dCh9WBMDrnFea973/8abNyLZUaVgly22K95Ofry8x0dG0btuOsRMm4XPvbrplL1+8gEqlok7dD9OdL9CjT79Uf1cqlXh73QagWs2aadapWr0G586e4fq1K3Tt2Subr0Rkm45DDJs3b2bFihUa10eNGsXo0VlbArls2TLWrl2LkZERS5cuxdraGgB9fX1UKlW6PRUpX95S/kzpncqt8unJ8wTh6dOn7N69m9OnT3Pv3j1iY2NT3Tc2NqZChQo0btyYXr16qX/AInu8L3jjfcE73fsfdkieURv2NIyIZ8mZckoPwtOHT2k9vDXVm9fE3Mqc5/7POL/3HBcOXtBoJ2VYwt/rMfU7NuDDDh9SrEwxXoZGcvPEdY5vPE5CXIK6vEKhoLhj8szfYL/gNGMLeZjcbWbnbKfdixbvncpVq7J5hxsVKlXKtOz9V+O1ZZ0cATh35jTux/8iKCgIKysrmjRrQbNPW6Z6s30aHExcXHLPmF3JUmm2W8Iu+ffU/3HWk2SRg3RMEAYMGECnTp00rltYWGRaNzExkVmzZrFr1y6MjY1Zvnw5devWVd83MTEhIiKCuLg4jc2TAPXvlqmpaao///v5qGt5k0xW2eRpguDm5sbcuXOJi4tDpVKhp6eHtbW1+gcWFxfHixcvuHnzJjdv3mT9+vXMnj2btm1ls6PcYFncks4Tk8+0P/3b3+rrDlWTVyOM2TQW08Kmqeo0G9Ccy39eZmH3H4mNev3LmLKCofWIz+kyqWuqOh93/ZgO33RiVpsZBNxLHtM1ty6s7rF4+TztLryXoZHqsgqFItPsV7y/qtesleWywUHJy2zNzMyZNO5r/j7pnur+sf8docHHDZm/aIl62WLYi1AAjIyM1G/G/2VRpMirsmFaxy9ygI7zCCwsLLKUDPxXVFQUY8aM4cyZM1hYWLBq1apUyQEkzw2IiIggJCSE0qVLa7Tx3zkKtra2eHl58ezZM8qVK5el8pD+HIOU8unNUUiRZ3MQzp49y7Rp0zAwMGDMmDEcPnyYW7du4eHhgbu7O+7u7nh4eHDr1i3++OMPxowZg76+PhMnTuTCBc1vrEI3xqbGfL9/KuZW5oSHhLNnvhsANvbFMLdKHuYJuh/EDy2n0c2sC72te7J04M9EPI+gzud1GLMx9Rhv2erJ38YSYhNwHbSMPkV70dWkMz98OpUHN/0oWb4k04/MxMQ8+Q3X2OT1tp/x/1n6+Pp6ctarp6eHoXHu7L4m3j/RUVEA7Ni2hbNnTjPi67H86f43J89fZO7CxVhaWXHe4ww/zZ+rrpPyDczYOO0x3jfvxcWl/S1O5DI9hW6PbAgPD6dfv36cOXMGOzs7tm/frpEcwOvVBffva67uioyM5OnTp1hbW2NjY5OqfMrqhDepVCp8fX3R19dXJw8ZlX/zedNbFZEizxKEtWvXYmhoyKZNmxg+fDjlypVDX19fo5y+vj7Ozs589dVXbNy4ET09vQyXkwjtFTIrxLQ/plOxfiWUiUqW9F1M2NPkbz2qpCT2/vQ7x379i8mfTOT6sWvERccR+SIS980nmNl6Bkqlko+7fULFV3MIAE5udefUtpN812QKxzce42XoS+Jj47l+/DpTGk0m5HEIduXs+HxkGwCSkjLvDVC8sZ5Xeg9ETomPT05Iw168YNjI0fQfNBhr66KYmJjSvGUrZs1fCMCfhw7g9+qNNSuz3JOSkrJcVuR/8fHxDB06FE9PT5ydndm5c2e6H8ANGzYE4PhxzQms7u7uKJVKGjdurFH+xIkTGuWvXr1KaGgotWvXVs/ZSynv7u6u/j1MERAQgJeXF6VKlcpwiSPkYYLg6enJhx9+qN5KMiuqVatG/fr18fJKexKb0J6FjQWzT8yletPqKJVKln2xlGt/XVXff/7kOZsmbmT5YNdUQwgp7l26y43jNwCo+8ZOiztm/MaSfovTXBoZFR7FkVV/Aqh3Z4yNjFHfNyqU9iEiKb0GSqUy1fwFIXSRMqRpampKr779Ne7Xq9+AylWqolKp8Djzt7osQHx8+it0EhLiX7Wffi+DyD1ve5mjq6sr169fx87Ojq1bt6r3PEhLq1atKFq0KPv27ePvv18P5z5+/JjFixejUCgYOHCg+nq9evUoX748Hh4e7N69W309NDSUmTNnAvDFF1+or9vb29OwYUP8/PxYtmyZ+np0dDRTp05FqVSmKp+ePJuDYGhomGaPQWYUCoU64xe6Ke5YnFl/zcbOuSSJCYks7b+E0ztPZ17xP/yu++LSygVbh4zHs97ke90XgGKv6sS8jCEhLgFDY0PMrc3VPRhvKly0MAAvn8t2yyLnmBdO/r1yKOuY7uE1juWcue35LwFPngCod2SMi4sjNiYmzS2Vw8OSdxe1srLKjbBFZt7iRklhYWFs3boVSN7+eN68eemWXbRoEebm5syePZuvv/6aYcOGUbduXczMzPjnn3+IiYlh3LhxVHpjgq2enh7z5s1jwIABTJs2jT179mBra8vFixcJDw+ne/fuNGvWLNXzTJ8+nV69erFmzRrc3d1xdHRUH9bUqFEjevXKfGVNniUIFSpU4J9//uH+/ftpTrpIi6enJ+fPn6dWraxPQBJpK1utLDOOzsLazprYqFgWdPuRK0cup1vewMiAxPjENO+lZNv/vW9UyIj42LSTuf/WUalUBNwLwKGqA7Zli+N/R3NzpZQEJNAnUOOeENnlUNYx0zJ6rz5sUvZBsC1eAlNTU6KjowkMDMDRSfM9LCgweQJuafsyORityLK3OLRz8+ZN9YoBT09PPD090y2bcupi8+bN2bp1KytXruTGjRuoVCoqVqzIwIEDad26tUa96tWr4+bmhqurKxcuXODevXs4ODjwzTff0K1bN43y9vb26vKnT5/m4cOH2Nvb079/fwYMGJBqT4/05FmCMHz4cAYNGkSvXr0YOHAgzZs3p1y5chpBJyUl4efnx4kTJ1i3bh1KpVK92YTIHjvnksw6NhvL4la8DH3JrDYz8f7nTppl+88fQMfxnbjpfpMZn6W9xbFjzeQ32MdejwGo06Yuk9wmo0xU0rdob/WOiG9y+k8dAJ/L93Co6kDF+hW5+r8rGnUq1k+e43A3g2WaQmirctXkzcP8fH2JjY1Nc3OZx4+Sh8pKvTHjvOIHlbl25TKeN2+mmSD8e/MmAFWqVcuNsEVm3mIPQqNGjfD21v59ycXFhV9//TXL5Z2dndW7MmaFnZ0d8+fP1zquFHk2B+Gjjz5i3rx5xMbGsnz5cjp27EiNGjX4+OOPad68OS1atOCTTz6hevXqtG3bliVLlhAfH8+0adNSTd4Q2jEyMWbaoWlYFrciPCSc75tMSTc5APC77oeBoQFVG1elWBnNbT7LVnekevMaKJVKzv3uAYDv1fsYGBlgWtiUBp0/0qhTyKwQLYd+BoCH2+uT9c7vPQdA037NMDBMnSiamJuod1D8e/sp7V60EBmoU+9DLK2siI2N4cDvbhr373nf4ca1qygUCho2ed2N26R58sE5B/fv1agTGBDApQvnMTQ0pNmnrTTuC5Ef5OlWy507d+bEiRMMGTKEDz74AH19fZ4/f86TJ0/w9/fn2bNnGBkZUa1aNUaOHMmff/5J79698zLkfK/7990pXckepVLJgm4/8uDWgwzLn993jkCfAIwKGTF5zxSKl319jKlznfJMPTgNfX19/rf6iHqDo9DAUPU+CsNWDKNq49ffoKxKWPH9gWkUL1sc32v3U33YXz58Gb8bfpRwKsHYzeMwMnk1eczClIm7J2FhU4Trx67hcyXtpTtCZIeBgQFDR4wCYJXrUo7+eVi9SiYwIICZ075DpVLxWZu2lCz1elOkth06UdTGhpvXr7Fs8U8kJib3lIU8fcqUCWNJTEykddv2FMtkrbnIJQodHwKF6h1bLxYeHk5MTAwKhQIzM7Mc32q5veL93WTJwMiALcHbMLc0J/plNA9u+GVY/seu8wkLDqNsdcfkIQlbSxITEgm4+wQ9fT1KV0o++vbioYv82GVeqqEEUwtTZh2bQ4V6yct8gv2CiHwRSZmqDuhZqNYAACAASURBVBgaGRLoE8D3Tb/jmX/qjTzKVivL3FPzKWxdmMgXkQTeD6RUxVKYFjbl6cOnTPjwG8KCZeOZzdH78jqEfOXKpYuMHDKIIpaWHD11VuO+SqViycL5uO34DUg+cMnSypr7PvdQJiZSuUpVlq7+BQuLIqnqXTh/jm/HjCI+Ph4ra2tsi5fA774P8fHxVKj0Ab9s3IyJSdobKYlkVia5s6fJgpabdKo/6a+BORJHfvbOJQi57X1OEJxrO7Pk8tIslx9cdhBPX21vnLzLYlfqtatHsTLFiIuJ4+GthxzfcIwTm9I+jMbAyIA2I9vSqHdj7D9ITiaC/YI5t8eDfYv2prlsEqBYmWL0mt6bWp+5UKRYEcKCw7h8+BK/Td8uycErkiBoJ7MEIcXZ06dw2/EbXp7/Eh+fQKnSpWn1eRu69+6b7sE3933useGXNVy9fImXLyOwLV6Cps1bMGjocMzkLJlM5VaCsPCzzTrVn/i/ATkUSf4lCYIQ+ZAkCKKgkATh3ZXnhzUJIYQQOU7mEehMEgQhhBAFj2xxrTNJEIQQQhQ8b3EfhIJKEgQhhBAFj+QHOsvTfRCEEEII8W6SHgQhhBAFj8xB0JkkCEIIIQoe6R/XmSQIQgghCh7pQdCZJAj5WNFSRek+tSe1W9fGys6KmIhovM7dYe+CPXid88pSG21GtWXY8uHMajuTy4cvaR2Dnp4erb/6nGYDm6t3S3zi7c/JLe78seIPkpRJadar06Yu7cd2wLmOMwqFAn+vx/y17ijuW9xRJio1yluVsGLIsqG4fFYbgGt/XePXb9bz7HFImu3PP72A4o7FGeY8hIS4BK1fl3h3hIeF0arJJxmWGTh4KMNHfa112zEx0fTq3BFLK0s2/bY7y/UiIsLp3aUjz0JC2Hv4aKozGlL8cWA/WzasJzDgCXYlS9G73wA6dtU8lhfg0oV/GD1sMOMnf0e3nnLeTE5QSIKgM0kQ8in7ymWYd2o+RYoVIS4mDn8vf4qWsubD9h9Sp00dVg5ZwfGNxzJsw6lWOfrN65/tGPT09Phu/1TqtasHQOD9QJSJShxrOlHOxRmX1nWY3Xamxgd+//kD6Do5+Y0y7GkYIQ+f4lCtLKN/HUPDno2Y12muxjbM3x+YRoV6FXj+5DkJcQl83PVjnOs4M6bGaKIjolOVrd26DlUaVmHlsBWSHBQAPvfuAlDE0hKHso5plilhV1LrdlUqFT/OnklQYACWVpZa1V3600KehaSdnAJ4nP6bOdOnom9ggKOTE48fPuTHOTOJi4ulR59+GuXXrHClhF1JOnZJO4EQIi9IgpBPjdvyDUWKFeGm+w0W9lhAxLMI9PT06DWzNz2m9uSr1SO4dfImwQ+C06xfvm4Fpv3xA6aFs3+QTOsRn1OvXT2iI6KZ22E2t07dAqBSg0pMPfQDLq1c6DyxC27zXn8z+7jbJ+rkYNvUreyZ70ZSUhJmlmZ8s20CddvUZeTaUSzus0hdx6WVCxXqVeDq/64wu90slIlKxm7+hmb9m9HiixYcXHYwVVx95/Ql0CeAY7/+le3XJt4dKQnCp599zoTJ3+VIm4kJCSycN4ejfx7Wuu4/5zz489CBDMtsWPcLAMvXrMOlTl387t+nf8+ubFj3C1179kZfX19d9m/3E3jeusnUGbMxNMydbYffS9KBoDOZxpEP2X9gj3NtZ5KSkljSdzERzyIASEpKYvu0bdy9eBdDY0Ma9WqkUVdPT482o9oy//SPWNpq963pv5r2awaA27zd6uQA4M75O/z2w3YAmg1onqpOj2k9Afhr/VF2z91FUlLyEERUWBRL+i7iZehLGvduoj4FEqBig0oAHN94XN0b8b81fwLwwceVU7X/cbdPKOfizI4Zv6U7vCHyl/v37gHgVK5cjrT3+NEjRgwZxMF9v2tdNyoqih9nz8A4nYObIDn5uHPbEwdHR1zq1AXAsVw5atWuQ3hYGA/9Xp+impSUxNpVK3Ao60jrdu21fzEifXoK3R5CEoT8qGipogC8fP6S0MBQjfv3r94HwKZM6nPoDY0NWXJlKcOWD8fAyIAdM39Lt4dBmzge3nqgGcMVHyD5ZMYUViWsKFutLAD7F2seNhQVFsXp3/4GoFHvJurr1nbWALx8HqG+lpIUmVmaqa/p6enRZ1YfHv77kL9ftSPyv/s+KQmCs85tHT54gN5dOnDz+jXKOJRlwKDBWtVfsXQxQYGBDP1qZLplXrx4gVKppEiR1Am4paUVAC9fvv49PnrkMPd97jHkq5GpehVEDlAodHsISRDyo2f+zwAoUqwI1iWLatwvU6VMcrlHT1NdNypkhFNNJx55PuT7JlPYMeO3HInDsaZTujGEPHo9TpuSLMRFx+J/xz/NNgPuBQCk6kGICosCkl9viiKvej9ehr5UX2s+sDmlK9mzfdo23rNDSgsslUqF3/3kZNMxBxKEu3e8UKlU9Ozbny073bB3KJvlulcvX2L/HjdqutSmY9fu6ZYzL2yOQqEg7MWLVNdDXyQn8xZFkn+PExMSWL96JRUqVqJ5y1bavxghcpkkCPmQ/x1/vDxuAzB28zgKF7VQ3+v8bReqNKxC9Mto3Le4p6qXEJfAkn6L+brGaDzPeOocx7H1yWP8XSZ1pUqjqurrTjWd6DMneSLWnys1x3gVenrpzjDWN0z+FlXM4XXPQ8qKjOZftEDfIPl+y8EtAbh9JvnnYGBkQI8fenHv0l3+2X9ep9cl3h1P/B8THR1NURsbXoQ+Z+Wynxk7YhhjRwxjxdIlPHr4UKv2GnzSkB17DzJ2wkQKmZhkuV5sbCzzZv6AkZERU36YkeEMeRMTU5zLV+DRwwdcuXQRgAd+vty4egUra2vKvEpKDuz7nSf+/gwbOVpm3OcGhY4PIZMU86t5neYyfvsEan5aiw2PNvDkbgBWJaywKmHFo9uPWP6lK8+fPE9VJz42nlPbTuZYDH+tP0oR2yJ0/747c0/OI9g3iMQEJSUrlCQ+Jp6t32/hkOvrCYTBfsnDGUaFjChZoRRPvDV7EcpUTu55MLcyV1+7fPgSvtd9qdXShfUPNhAfG49dOTsCfQI4sek4AK2Hf46tgy0rhizPsdcn8l7K/IPIyEj6dO2EUvl6Rcw/5zzYuW0L4yd/R6cMvtG/qf5HH2crjjUrXPF//JivRo/Boawj0dHRGZYf8OUQpk6awJivhlLWyYnHjx6RkJCgHkqIjY1l07q1VKtRk48bNc5WTCITMo9AZ9KDkE8lxCXg/Y83cdGxGJsWwqmmE1Ylksc4QwNC39ryvoC7Twj2C0ZPTw8755LYf2CPvr4+0RHRvHz+MlXZ8JBw7l1KnpHeY2oPjbZsStvQ8NXESgOj17mrMlHJ9JbTOLX9FIXMC1HEtghnd59hSuMpxEbFYmxqTLfvunHr1C2uH7umrmdmaaaevyDyp5QVDHGxsbTv3IVd+w9x5tI1dh84TKeu3UhMTGTh3NmcPX0q12L49+YNdv+2jfIVKtJnwBdZqtOi1WfMmr+Q0mXK8MDXl+LFS/D9jFl07pb8e79n52+EhDxNtXdDYmIiwUGBxMbE5MrreO9ID4LOpAchHzIrYsbck/NwqlWOq0evsnnSRh7ffkzRUkVpN7YD7ce0Z/7pH/nh02l4/3Mn1+Lo+UMves/sQ1jwCxb2WMDlw5fQ09ejTpu6DF46hBFrRlKyQik2jF+vrrNt6jamH5lBk75NiQqL4veFvxMW9IKK9SsyfNUI4qJiMTYxRpmQeu+E8JBwlvRd9N8QAOgwriOWxa2Y33kekDypcdzW8dRoXgNIntewavhKbrrfyKWfhMgtFT/4gE5du+HkXD7VBkJlHByYNHU6+gaG7Nn5G8uXLOaTRk1y/Pnj4+OZM2MaCoWC72fMwsAg62+ZLVt/TsvWn2tcj4qMZOumDdSr34DadZP3ENnrtovVrst4+TICAwMDWrdrz4TJ32NsbJxjr+W9I8M2OpMehHyo88QuONUqx4ObfsxuOxO/G34kJiQS/CCY9WPXsnfh75iYmzB85fBci6FUxdL0+KEnSqWSuR3ncnb3GWKjYomOiOb0jr/54dNpJCYk0vGbjpSt/npzm2t/XWXt6F9QJippM6otGx5tZG/8fuafXoBRIUNWDlsJoLH5UXrMLM3oOKETlw5fUs9VGLd1PNWaVmP/kv2s/molRiZG/HB4OqUrlc75H4TIVZ80asKkqdPT3V1w4JdDAHj4wE/r+QhZ8esvq3ng60vPvv2pVLlKjrS5fcsmwsPCGDYyuffg+tUrLJw7G5tixZg8bQYtW7fh0L69/Dh7Ro48nxDZJQlCPtSgy0cA7P1pb5rbErvN340yUUk5F2fsytnlTgydGqCvr8+/p/5Ns5fiwU0/Lh68AMDH3VKP+/656jBja33NIdeDXD92jfN7z/HL6DWMqfk1US8iAXgR+EKjzbR0mdQVUwtTtk/dCiTvDlmjeQ1O7zjNhvHrObLmCKuGrcCokBHtxnTQ5SWLd5BNsWJYWScPIwUFBuRo23fv3GHb5o2UtrdnSAbLGrUR9uIFO7dvpXHTZlSpVg2AbZs3ArDgZ1c6dunKD7PnUrlKVY7+eZiQp08zak5kQKGn0OkhZIghX7J1SN7fIL2lglFhUYSHhGNtZ00xB1sC7wfmeAzF1DE8TrdMwN3kN+yUeN/08N+HrBuzVuO6U61y6vuZsSxuSdvR7Ti3xwPf674AlHNJru99/nXScufVv5evWz7TNsW7JzEhAYWeXvr7BLxa0ZrTuxD+ffIEysRE/B8/pkn9OumW69wmeYni1JlzaNuhY4Ztbvp1HbExMQwdOVp9zdvLiyKWlpRxcFBfq1K9Orc9/+WO122K2Wr+/yOyQD7jdSYJQj4UHRGNUSEjrO2s0rxvaGyo3jMgq1312op51W5GkwBTliq+GUPDHo0o7licY7/+RXhIuEadOm2Td577942dGdPTY2pPDI0N2f5q10ZIThqAVJM0U57f1CL720qLvNG+VXOeBgcza/7CNMfzQ54+5cWr/QUcHNM+pyG7StjZUb1mrTTvqZKSuHUzeU7LB5WrYGhkhHVRzT1J3vQ0OIh9brv49LPWlHN+nayGhj7H2ir1/0dmZsmreKIiI3V5Ce83mYOgM0kQ8qF/T93ik+4N+fTLllw8dFHjfpO+TdE30CcyLJIHN/zSaEF3t07dovPELtRq5YJ1yaKEBqReUlmkWBFqtXJJjvfvf9+IrQl129Yj8kUk//vlSKo6FepVoHrT6rwMfcmZXaczfH5bB1taDmnFya3uqZZLpqycsC75+g03JYlJ2X1R5B9O5Zx5GhzMn38cTDNB+G3rZgBq1a6DtXXGH9DaatexM+06dk7zXnR0NM0+Sp5gOPenJWme5vhf69esJlGpZPDw1MMVRYoU4cWLUBITEjB41Qvy/FnyBmOWVml/CRBZIMMEOpM5CPnQ7wv2kJiQyIcd6jNwwRcYm76e6fxRl48ZtPjL5HI/JpfThb6BPqUqlqZUxdIYmbx+nitHLnPv8j0KmRVi2qFplKrw+g3S1sGW7/Z9j0VRCx7++5Dze8+p753ZdQaA3rP64FD1dZeqc21nJrlNBmDPfDdiXma81Kv3zD4AGrtBpmwg1bhPE3WPwWfDWwPgefpfRP7Su/9AAP7xOMsq159JSEjuGVIqlWzfvJGd27agr6/PyLHfqOs8CwnhgZ8vQYE5P7SWXY8fPeLwoQO069AJ+zJlUt2rXrMWiYmJHHh1NsTT4GDO/v03xsbGVK5aNa3mhHgrpAchH7p/9T4rBrsyct1oOk/swmdftSbgbgA29jbqA5hObDrO7wv26PxcRUsVZfWdNQB812QK//79uuv/xy7zmH18LuVcnFnptRr/O/7o6SmwK18SfX19gnyDmNthdqpDk05tO0n9jvX5qMvHLL3mmlxHXw/7D+wBOLr2f+xbtDfDmEpXKk3jPk3435ojqbZyhuS5C+f3nadBpwas9V1PREg4pSvZExb8ggM/79f55yHernr1GzB81NesWeHKlg2/snf3bkqXsScoMJCwFy/QNzDg++kzqVqturrOKtel/HnoALVq12H1r5vyLvg3rF21HAN9fQYNHaZxr/8XX3L271Msmj+X33ftJCgokOioKAYNHY6FRZE0WhNZIh0IOpMEIZ9y3+KO73U/Ok3oRLWm1XGo5kBsZCzXj13j6Nr/4bHHI9djCHkUwrjaY2g/tgMfdfmYkuWTV0z4e/lzfu85DizZT1R4lEa9n3oupP3YDjTt34yS5UuiTFTiefpfjqw5wukdmR+y1HdOPxLiEtg1Z2ea9xf3/okBCwbSuHdjipUpxpUjl1k3Zi1hwWG6vWCRJwYOHkq1GjXZuX0rt25cx+fuXaysrGnZug39Bn5B+YqV8jrEDN27683xo/+jR59+2BYvoXG/ctVqLFu9lhU/L8bn3l2K2hTjy6HDs7wpk0iHzEHQmUL1np1q017RNq9DEEJnm6M1T8MUIj+yMsnZ1ScpFg3JuCcyMxPWpT3/5H0icxCEEEIIoUGGGIQQQhQ8MsSgM0kQhBBCFDhyhLbuJEEQQghR8MgAus4kQRBCCFHwSA+CziTHEkIIIYQG6UEQQghR8EgPgs4kQRBCCFHwSP+4ziRBEEIIUfBID4LOJEEQQghR8EiCoLNME4QPPvggWw0rFApu376drbpCCCGEyFuZJggVKlTQ2HAiMDCQ8PBwChcuTJUqVShSpAjR0dF4eXnx7NkzSpYsme3EQgghhNCZzEHQWaYJwoEDB1L9/fLlywwaNIjBgwfz9ddfY2RkpL6XlJTE+vXrcXV15Ycffsj5aIUQQoiskCEGnWmdY/30009Ur16dCRMmpEoOAPT09Bg6dCgffvghS5cuzbEghRBCCK0oFLo9hPYJwp07d6hWrVqGZZydnfHz88t2UEIIIYTIW1qvYihatChXr15N935iYiLnz5/Hzs5Op8CEEEKIbJM5CDrT+kfYtm1bbty4wbRp0wgNDU11LygoiPHjx3Pv3j26deuWY0EKIYQQWpEhBp1p3YMwcuRIrl27hpubG7///jt2dnaYmZkRGRlJYGAgKpWKzz77jEGDBuVGvEIIIUTm5ENeZ1onCMbGxmzZsoW9e/dy+PBhvL29CQ4OpkiRIjRs2JBOnTrRunXr3IhVCCGEyBoZYtBZtnZSVCgUdOnShS5duuR0PEIIIUS+t3fvXqZMmcL27dupU6dOqnuBgYE0adIk3bouLi7s2LEj1bXg4GBWrlyJh4cHISEh2NnZ0b59e4YMGaKxohAgIiKCX375hePHjxMYGIiNjQ0tW7Zk1KhRmJubZ+k1ZHur5cTERDw8PLhz5w7h4eFMnDgRb29vTE1Nsbe3z26zQgghhO7ycIjh2rVrzJ49O937KbsMV6xYkQoVKmjcd3R0TPX3oKAgevToQVBQEJUrV6ZKlSpcvXoVV1dX/vnnHzZs2IChoaG6fGRkJH379sXb2xtHR0eaNGmCp6cnGzdu5MyZM+zcuZPChQtn+jqylSBcuHCBSZMmERwcjEqlQqFQMHHiRI4cOcK6dev45ptv+PLLL7PTtBBCCKG7PMoP/vrrLyZPnkx0dHS6Zby8vAAYPHgw7du3z7TNGTNmEBQUxJgxYxgxYgQA0dHRjBw5knPnzrF169ZU8/6WLl2Kt7c33bt3Z+bMmejp6ZGYmMh3333HgQMHWLp0KdOmTcv0ebUepfHy8mLo0KHExMQwbNgwWrZsqb5Xo0YNbGxsWLRoEe7u7to2LYQQQuQMPYVuDy0FBQUxceJERo8eTVJSEjY2NumWTelBqFKlSqbt+vr6curUKcqUKcPw4cPV101NTZk7dy76+vps27ZNfT0iIgI3NzfMzc2ZNGkSenrJH/MGBgZMnz6dIkWKsGfPngwTmBRaJwiurq4YGxuzd+9exo4dm6p7pGnTpri5uVGkSBE2btyobdNCCCFEvrR06VIOHDhA1apV2bVrF05OTumW9fLywtTUVGMoIS1nz55FpVLRtGlT9Yd9ipIlS1K5cmWePHmCj48PAJcuXSI2Npb69etrzDUwMzOjQYMGxMbGcunSpUyfW+sE4cqVK3z22WeUKlUqzfu2tra0bt2ae/fuadu0EEIIkTPe8j4ITk5OLFiwADc3NypWrJhuubCwMAICAnB0dGTjxo20b9+eGjVq8MknnzBt2jSCg4NTlU/54C9fvny6zwtw9+5drcp7e3tn+pq0noMQFxeHqalphmX09fWJi4vTtmkhhBAiZ+g4ByEiIoKIiAiN6xYWFlhYWGhcHzp0aJbaTZl/4Onpyd27d6lbty4lSpTg1q1b7N69m5MnT7Jlyxb1B/nTp0+B5C/faSlWrBgAz549AyAkJCTV9fTKP3/+PNNYtU4QypUrh4eHB0lJSRrdHQAJCQmcPXs2S10nQgghRK7IxjyCN23evJkVK1ZoXB81ahSjR4/Odrsp8w/Kly/P6tWr1av+oqOjmTZtGn/88QcTJkxg7969AMTExABQqFChNNtLuZ4ypyDlTxMTkyyVz4jWCUK3bt2YOXMmkydPZsqUKanuPX/+nFmzZvHw4UO+//57bZsWQgghcoaOyxwHDBhAp06dNK6n1XugjYEDB9KyZUvMzMywtrZWXzc1NWXOnDlcunQJT09Prl+/Ts2aNdVfxBXpvB6VSpXqT23LZ0TrBKFXr15cu3aNgwcPcujQIYyNjQFo1qwZQUFBJCUl0aJFC/r06aNt00IIIcQ7Ib2hBF3p6+unu1eQiYkJ9evX58CBA3h6elKzZk31kH5sbGyadVKG81N6DLQtn5Fs7YOwcOFCmjZtyp49e7h9+zaJiYlERkZSu3ZtOnXqROfOnbPTrBBCCJEz8ulRDCnLI1OGFlLmHqTMMfivlDkHKeWyWj69OQpvyvZOiq1bt87wzIWYmJgsZShCCCFEjtNxDkJuWbFiBXfv3mXkyJFprnbw9/cHoESJEsDr1QgpqxP+6/79+wDqLQeyWj6jlRYptF7m2Lx5c7Zs2ZJhmRUrVtCsWTNtmxZCCCFyxjt63LO3tzdHjx7lyJEjGveeP3+Oh4cHhoaGfPjhhwA0bNgQAHd3d5KSklKVDwgIwMvLi1KlSuHs7AxA3bp1KVSoEOfPn9eYiBgVFcX58+cxNTWldu3amcaaaYLg7+/PnTt31I8nT57g6+ub6tqbj1u3bnH+/Hl194gQQgjx1il0fOSSHj16ALBx40auXLmivh4VFcV3331HZGQkXbt2VQ8B2Nvb07BhQ/z8/Fi2bJm6fHR0NFOnTkWpVPLFF1+or5uamtKxY0fCw8OZOXMmiYmJQPL5SbNmzSIiIoIePXpk6cCmTIcYbty4wfjx49UzIhUKBbt27WLXrl3p1lGpVHz88ceZPrkQQgjxPvnkk0/44osv2LhxI3379sXFxQUrKysuX77MixcvqFOnDpMmTUpVZ/r06fTq1Ys1a9bg7u6Oo6MjV69eJSQkhEaNGtGrV69U5ceNG8eFCxfYv38/V65coXLlyty+fZvHjx9TuXLlLC/TzDRBaNOmDbdv3yY0NBSVSsX+/fupVKkSH3zwQZrlDQ0NsbW1lVUMQggh8s47OgcBYPLkydSoUYNt27Zx+/ZtkpKSKFOmDIMHD2bAgAGpTmaE5F4ENzc3XF1dOX36NA8fPsTe3p7+/fszYMAADAxSf5RbWlqyc+dOVqxYwfHjxzl58iR2dnYMHjyY4cOHY2ZmlqU4FaqsLIZ8Q7NmzRg4cCD9+/fXpto7o72ibV6HIITONkfvy+sQhMgRViaGmRfKhsUL/9ap/viJjXMokvxL61UMKac0hoaGEh8fr55pCXDw4EHq16+f7paQQgghxFuh9RR88V/Z+hGuWLGCRo0aceDAAfW1hIQEJk+eTLNmzdi6dWuOBSiEEEKIt0/rBOH3339nxYoVlCtXjurVq6uvKxQKfvzxRypUqMC8efM4ePBgjgYqhBBCZNk7uswxP9E6Qdi+fTvlypXDzc2NBg0aqK8bGBjQvn17du7cSZkyZdi0aVNOximEEEJknSQIOtM6QXjw4AGNGzfGyMgozftGRkY0bdo03V2chBBCiFynp+NDaD9J0dTUlMDAwAzLPH/+XLZZFkIIkXekF0BnWudJdevW5fjx49y8eTPN+3fu3OHYsWNZ2sZRCCGEEO8mrXsQhg8fzsmTJ+nfvz8dOnSgZs2amJubExUVxc2bN9m/fz8qlYpRo0blRrxCCCFE5qQHQWdaJwgVK1Zk9erVTJ06lV27drF79271PZVKhZ2dHfPmzaNy5co5GqgQQgiRZTKPQGfZOu65QYMGHDt2jOvXr3Pnzh0iIiIwNTWlQoUK1K1bF319/ZyOUwghhMg66UHQWbYSBAA9PT1cXFxwcXHJyXiEEEII3UmCoLNME4QtW7ZQs2ZN9aZIW7ZsyXLj7+J5DbsSZAMnkf/Je58QIrdlmiDMmzePUaNGqROEefPmoVAoyOyMJ4VC8U4mCEIIId4DMgdBZ5kmCPPnz091tPP8+fNzNSAhhBBCVwrpZtNZpglCp06dMvy7EEII8c6RBEFn0gkjhBBCCA2Z9iBkdx6BQqFg8+bN2aorhBBC6EI6EHSXaYJw8eLFjBswMKBw4cLExMQQGxsLgLGxMcbGxjkToRBCCKElmYOgu0wThEuXLqX6e0BAAF9++SVOTk5MmDCBqlWroqeXPFJx7949Fi9ejJeXlxz3LIQQIu/IALrOMv0RFi5cONVjxYoVmJubs3btWqpXr65ODgDKly/P8uXLsbCwYM6cObkauBBCCJEehUKh00NkI8fy8PCgUaNGFCpUKM37hoaGfPTRR1y9elXn4IQQQgiRN7TeatnExITAwMAMy/j4+FC4cOFsByWEEELoRHoBdKZ1D0KDBg04ceIEBw4cSPP+pk2bOHfuHM2bN9c5OCGEECI7FArdHgIUqsz2TP6PjspJeQAAIABJREFUJ0+e0KNHD54/f46zszNVq1bFzMyMyMhIrl27xqNHj3BwcGDnzp1YWlrmVtzZFpOYlNchCKEzeQMTBUUh/dyZTei65YpO9b/uXzuHIsm/tB5iKFWqFL///jtLlizh2LFj3Lt3T32vcOHC9OzZk3HjxmFhYZGjgQohhBBZpdCTLFpXWvcgvCkhIYFHjx4RERGBhYUFDg4OGBhk+wTpt0J6EERBID0IoqDIrR6E5dt0myg/uq9LDkWSf+n0XyY+Pp6IiAjCw8MpV64cUVFRORWXEEIIkX0KHR9C+yEGgGfPnjF37lyOHTuGUqlEoVBw+/ZtfvvtN/bu3cv8+fOpU6dOTscqhBBCZInsZaA7rXsQQkND6dGjB0eOHKF69epUrlyZlFEKExMTAgICGDJkCN7e3jkerBBCCJEVsopBd1onCK6urgQGBrJ69Wp+++03mjZtqr43cOBANmzYQGJiIqtXr87RQIUQQgjx9mg9xODu7s6nn36aKjF404cffkjLli25ckW3JSZCCCFEtkk3gM60ThBevHiBvb19hmWKFy9OaGhotoMSQgghdCFzEHSndYJQokQJbt++nWGZmzdvUqJEiWwHJYQQQuhETnPUmdY/wlatWnH+/Hl27tyZ5v2NGzdy5coVWrRooXNwQgghRHbIaY6603qjpMjISHr16oWPjw/Ozs4kJSXh6+tLhw4d8PT0xMfHhzJlyuDm5vZO7qYoGyWJgkDev0RBkVsbJa3ec1On+l91rZ5DkeRfWv+XMTc3Z8eOHfTs2ZMnT55w//59VCoV+/fv5+HDh3To0IEdO3a8k8mBEEKI94Ssc9SZ1j0I/v7+lC5dGgClUomfnx8RERGYmpri5OSEkZFRrgSaU6QHQRQE8v4lCorc6kFYs/eWTvWHd66WQ5HkX1pPUuzfvz/VqlVj2bJl6Ovr4+zsnBtxCSGEENkm8wh0p3WC8OzZs0yXOQohhBB5SlYx6EzrH2HdunU5d+4c8fHxuRGPEEIIId4BWvcgdOvWjTlz5tCqVSsaNWpEqVKlKFSoUJpl+/fvr3OAQgghhLZkiEF3Wk9SrFSpUtYaVijw8vLKVlC5SSYpioJA3vtEQZFbkxTXHsp4Q7/MDG1XOYciyb+07kGYP39+bsQhhBBC5BhJonWX5QQhMDCQs2fP8uLFC4oXL06jRo2wsrLKzdiEEEIIkUeylCAsW7aMdevWoVQq1dcKFSrEpEmT6NmzZ64FJ/7f3n3H13T/Dxx/3SRkihWxV4SUEiKoUTsoiqIUrRJij5Z+jRoxolqjFbEVpUTtTc1YtYVWkCESM4IM0uzc5P7+iNyfuDfzJsLt++lxH/Sczzn3c9JPbt55f5YQQohckRSCzrIMEPbt28eKFSswNTWlY8eOlC5dmgcPHuDl5cWsWbOoXLkyTZo0eRt1FUIIIbJFYSABgq6yDBDS9lTYsWMHlSpVUh/38fHhq6++wtPTUwIEIYQQ7xRJIOguywAhICCATz75JF1wAFCnTh1atWrF9evX861yQgghRK5IhKCzLOeXxMTEULJkSa3nqlSpQmRkZJ5XSgghhBAFK8sMglKpxNDQUOu5QoUKoVQq87xSQgghhC5koSTdyWrVQggh9I9Cx5eOdu3ahZ2dHVevXtV6Pjg4mPHjx9OyZUvq1q1Lly5d2LRpEykp2hfze/r0Ka6urrRt2xZ7e3s6dOjAsmXLMtz2ICoqigULFtChQwfs7e1p06YNP/30E9HR0dl+BgkQhBBC6B2FgUKnly6uX7+Om5tbhuf9/Pz4/PPPOXjwIOXKlaN58+aEhobi5ubGxIkTNcqHhobSu3dvtm7diqWlJa1atSImJgYPDw8GDx5MUlJSuvLR0dF89dVXrFmzBoVCQatWrVAoFPz222988cUX/Pvvv9l6jmytg+Dn58eePXs0jqctpaztHMBnn32WrUoIIYQQeamgOhiOHj3K5MmTiY2N1XpepVIxceJEoqOjmT9/Pt26dQMgIiKCgQMHsn//ftq1a0eHDh3U18ycOZPQ0FC++eYbRo4cCUBsbCyjRo3i/PnzbNy4kUGDBqnLu7u74+/vT+/evZk1axYGBgYolUqmTJnC3r17cXd3Z/r06Vk+S5Z7MXzwwQcZ9uWkXfrmeZVKJXsxCJGPpHtV6Iv82oth/Yk7Ol0/sG31HJUPDQ3ll19+Ye/evZiammJubk5YWBienp40aNBAXe6vv/5i8ODBNGrUiI0bN6a7h7e3N/369aNhw4Zs2rQJgKCgIDp16kTFihU5cuQIBgb///UKCQnBycmJMmXK4OXlBaR2LTRv3hwjIyNOnz6NhYWFunxMTAytW7cmISGBCxcuYGZmlukzZZlBGD16dDa+NEIIIcS7420PUnR3d2fv3r3Url2buXPnMmfOHMLCwjTKnT17FgAnJyeNc46OjpQsWRJvb2+io6OxsLDgr7/+QqVS0bp163TBAUC5cuWoVasWPj4+BAYGYmtry5UrV4iPj8fJySldcABgbm5OkyZNOHz4MFeuXKFly5aZPpMECCJT9T6sme2y/b7qz8TvpwCwYtlSVi1fprWcoaEhxsbGlC5ThkYfNebrgc6Ur1AhT+orxJtWLF3KygzaYmYOHTvOvt27M7w2XTtunNqOK0g7fme87SybjY0N8+bNo2vXrho/yF8XGBgIQI0aNbSer1q1KuHh4dy9e5e6deuqy1evrj2jYWNjg4+PDwEBAdja2marPIC/v7/uAYIQALbVq2NhUSTTMhUqVtQ4ZmFhgW319N8IKSnJRP8bzYMHDwgOCmLfnj14LF9Bw0aN8rTOQgCUKVuWevXraxy/ffMmiYmJVKpcmRJa1noxLlxY/W8LCwts3/hAT0l+ox3v3sOSFdKO3xVvO0AYOnRotso9e/YMgFKlSmk9n3Y8LfuQVt7a2jpb5Z8/f56t+4eHh2dZVwkQRLZMmjItVx98djVrsnb971rPhT55wvhvxnD71i2mfT+JvQf/xMTERNeqCpFO95496d6zp8bxjk5tCQkJwWXoMLp1757pPT6oWZO1GzJux+PGprbjqZMnse+QtGN9EBUVRVRUlMZxS0tLLC0tc33fuLg4gAzbSNrxtEGOOS2f9repqWm2ymdGpjmKAlOmbFl++Gk+CoWCp6GhnD7pVdBVEiLHypQty9x5/9+OT0k7ficodPyzYcMG2rZtq/HasGGDTvVK637IavB/2t/5XT4zkkEQBaqqjQ2VKlfm/r173Lp5kw4dOxV0lYTIsXTt2Ocmn0g7LnC6djEMGDCA7loyS7pkDwD1zIH4+Hit5xMSEtKVy275tIxBTstnRgIEUeDSxjbExMQUcE2EyD2LItKO3yW6Bgi6diVkxNraGl9fX8LCwqhWrZrG+TfHEKSNPdA2I+L18mnlsls+ozEKr5MuBlHgHj18AECZMmUKuCZC5N6jB6/acVlpx+8ChUKh0yu/pM0uSJtt8DqVSkVQUBCGhobq4CGz8gB3794F/n9WRHbL29nZZVlXCRBEgTq4fx8vX74E4OMWLQq4NkLkzoF9/9+Om0s7Fplo3rw5ACdOnNA4d+3aNSIiInB0dFSvYZBW3svLS2OfhpCQEHx9fSlfvjy2trYANGzYEBMTEy5cuKAxEDEmJka9QJKjo2OWdZUAQWTLEOcB1PuwZoYvrxPHs30vpVJJSMhjNm/ayFy32QC0cXKiZq0P86v6QuQ5pVJJyOPHeG6UdvwuKuC9mjLUqFEjqlevzrlz59i2bZv6eEREBLNmzQLA2dlZfbxixYo0b96c4OBgFi9erD4eGxvLtGnTSE5OTlfezMyMzz77jJcvXzJr1iz1jstKpZLZs2cTFRXFF198obGIkjYyBkFkS1brIBQrVkzrce8rV7JcbOmjJk2Y6faDTvUTIj9dvXKFurUyb8eNmzRh1hxpx++Kd3W7ZwMDA+bOncuAAQOYPn06O3bswNramsuXL/Py5Ut69+5NmzZt0l0zY8YM+vbty8qVK/Hy8qJq1apcu3aN58+f06JFC/r27Zuu/Lhx47h06RJ79uzB29ubWrVqcfv2bR4+fEitWrUYM2ZMtuoqAYLIltyug6BtoSQjIyOKWBahalUbmn78MQ0aysIy4t2mbaEkIyMjihRJbcfNmks7fte8o/EBAPb29mzfvh0PDw8uXbrEnTt3qFy5MuPHj6dXr14a5StWrKguf+bMGe7fv0/FihX5+uuvGTBgAEZG6X+UFytWjC1btrB06VKOHz/OyZMnKVu2LC4uLgwfPhxzc/Ns1VMCBJGvMlsoSYj3RWYLJQmhzZsbMb3J1tYWDw+PbN+vbNmy/Pjjj9kuX6xYMaZNm8a0adOyfc2bJEAQQgihd97hBMJ7QwIEIYQQeuddHYPwPpEAQQghhN6R+EB3EiAIIYTQO5JB0J2sgyCEEEIIDZJBEEIIoXckf6A7hSo7ez7qkThlStaFhHjHSfZU6AsTw/xJZO+5dF+n6z/7qHIe1eT9JRkEIYQQekfGIOhOAgQhhBB6R8ID3ckgRSGEEEJokAyCEEIIvSM9DLqTAEEIIYTekTEIupMAQQghhN6R+EB3MgZBCCGEEBokgyCEEELvKGQeg84kQBAagoOCWL9uLZcvXSTs+XOMTUyws7Oje8/P+bRrN63X3L51i/Xr1nDt6lVevHiJlZUVH7dowbCRIylVylrrNefPnWOL5yZ8fG7wb9S/FC1alLoO9fh64CDqOTjk5yOK/5iUlBT69+3Lo4cPOH3+QrauSUpKom+vz7kTEMCa9Rto2KiR1nKH/zzEzm3b8fP1JTY2lhIlS9CwUSMGuQzBtnr1vHwMkQPSxaA76WIQ6Zw+eZI+n/dg7+5dRISHU7WqDcaFjfG+epVp30/m+4kTeHPxzd07d9C/7xccPXwYgKo2VQkLC2PHtq30/bwnjx4+1HifpYvdGTnUhTOnT6FUKqlma0tCQjxex48z6Ouv2Lblj7fyvOK/Yenixdz0uZGja9asWsWdgIAMz6tUKqZOnsSk777j8qWLGBUyolq1akRFRXFw/3769e7FyRMndK26yCWFQreXkABBvCY8LIwpkyaQkJBAj897cercBbbt3sOJM2f5xWMJ5ubm/HnwAH94blJfc/vWLdxmziAlJYX/TZrM0ZOn2b57L4eOHce+Xj3CwsJwm+ma7n3+OnuGNatXYWRkxFTXGZy9cImtO3dx6twFRowaTUpKCvPm/oDv7Vtv+0sg9IxKpWLlsmWs/XV1jq67ExDAmtWZX7N96xYO7NuHhYUFPy9ezMmzf7Ft925O/XWOz3t/QUJCAt9PnMizZ890eQSRSwYodHoJCRDEa3bt3EFMTAw1a9Vi2oyZmJqaqs+1aevEmG/HAbDp9w3q478smE9KSgrOg1346usBGBikNilra2vm/jQfhULBpYsXCQl5rL5m44b1AHzRtx+9vuijPm5kZMSwkaNwat+B5ORk/vD0zM/HFXou7Plzxo0Zw4plS3N0XXJyMjOmTQVUGBkVyrDcxg2p3wcjx4zBqV179XFTU1OmurpSu449cXGx7Nq+PVf1F6KgSYAg1K5euQxAG6d26h/0r2vRqhUAIY8fE/XyJU9DQ/G+egVzc3MGDx2mUb5CxYr8b9JkJk2Zqv6gTUlJ4e9r1wBwat9Baz1atEx9H7/bt3V9JPEfdf7cObp26shJrxNYWVkxdtz4bF+74bd13Lp5k/4DB2JhYa61zNOnT3lwP3UzIG3t2MDAgOYtWgDgK+24QEgXg+5kkKJQGzVmLJ0/7Uqt2h9qPR8XG6f+tzI5mcuXLqJSqWj4UWPMzbV/kH7Z/+t0/52SksJC98U8DQ3NcABXXFzsq/dQ5uYxhCDobiCxsbF82rUrEyZP5k7AnWxddy84mJXLllGpcmVGjBrN7h07tJYrYmHB4mXLefY0FGtr7YNw09pxsrTjAiE/5HUnAYJQs69bD/u69TI8f+qkFwDFS5SgePHiBN5J/dC1sbEB4OyZ0xw/epTQJyEUL16CNu3a0a59h3QrmhkZGdG8RctM63HKK/V9bKrZ6vQ84r+rdh17tuzYyQc1a2b7GpVKxYzp00hMTMR11myMjY0zLGtmbk6r1q0zPJ+SksKZ06cBaccFRVZS1J0ECCJbwp4/Z/26tQB07NQZhUJBaOgTAMwtLBg3drTGiO3Dfx6iWfPmLFy0ON14hsycO3uWC+fPAdDp00/z8AnEf0lupslu3rSJv69do2ev3hlOacyuHdu2EXT3LgAdO3fW6V4idyQ80J2MQRBZiouNZdzY0fwbFUXx4sUZPGQoADExMQBs2rCes6dPM/bb8Xid+YsLV7yZ/8siihcvzrmzZ/lxjlu23udecDDTvp8EgGODBrRu0zZ/HkiINzx69Igl7u6UsrZm3P/+p9O9rnl7s3DeTwB82rUrNWvVyosqCvHWFXgG4dqrAWu5Vb9+/TyqidAmNiaGsaNG4nPjBoaGhvzw03xKWlkBkJiQCEBkZCTfjBuPs8sQ9XXtO3yCpaUlw10Gs3/vHgY4D6Kabcap1nvBwQwd7ExkZCRWVlbMnbdAUoTirZntOp24uFjmzptHkSJFcn2f69euMWbEcBISErCpVo0p012zvkjkC/n80F2BBwj9+vXL9f9IhULBbRkhnG8iIiIYO3IEN31uYGBgwKw5P9D044/V541NUvtozczM+OrrARrXN27SlA9r1+HWTR/OnjmdYYBw08eHMSOHExkRQbFixVi+eg2ly5TJn4cS4g07t2/j0sWLtO/wCW2cnHJ9n7OnTzNh/Hji4mKpULEiK39dk+HgXZH/JD7QXYEHCHPnzmXOnDnExsZiZWVF1apVC7pKAnj08CEjhrjw8OEDjIyMcJv7k0ZfapEilgBUqWpDocKFtd6nmq0tt2768PjRI63nz545zYTx44iPi6NkSStWrllL9Ro18vZhhMjA06dPWbRwIZaWRZk8dWqu77Nr5w7mzJxJcnIyVapWZfXadRLkFjAJEHRX4AFCjx49qFq1Ki4uLsTExODq6kp1Wb+8QAX4+zNyqAthYWGYmJqy4JdFWmceVKlSJct7GRikfpcaFdJccObQgQO4Tv0epVJJhYoVWbF6DRUrVdK5/kJk18Xz5/n3338BaNOieYblXAamZsiGjxzFiNGj0537be0a3H/+GYAPa9dm6cpVlChRIp9qLMTbU+ABAoCDgwM//vgjY8eOZerUqWzbtq2gq/Sfdf/+PYYPGUxEeDiWlkVZsmIFdetpHxFeu449AEFBd4mPj8fExESjTNpiMhUqVEh3/MSxo0yfMpnk5GRq2H3A8lWrsSpVKo+fRojMlSxZknqZjGO6eeMGSqUS2+rVsShShDJly6Y777nxd3Vw0LhJExZ5LMFMuhXeCbKbo+7eiQABoH379nTt2pX9+/ezd+9eunXTvmugyD9xcXF8M2okEeHhFC9enFVrf6OGnV2G5Rs1bkzx4sWJjIxk5/ZtGosi+fv5cf3aNRQKBa1em5EQGHiHKZMmkpycTO069ixf/SuWlpb59lxCZOTjFi34+NWKh9q0bNqEFy9eMHnqNI2pjxfPn2fBT6mzFVq0asXP7ospnEFXm3j7pItBd+/UNMfvvvuOzz77TDY3KSBrVq/iXnAwBgYGzP/FPdPgAFIXPRo5ZiwAHot+4dCBA+qdHkNCHjP9+8moVCo6d+lC+fLl1de5zZhBQkICVlZWeCxbLsGBeO8kJSUxy3U6KpUK2+rVWfDLIgkO3jEKhUKnl3iHMggApUuX5scffyzoavwnJSYmsvWPzQCYmJiwzGNxpuUXLnLHqlQpPu/9BUF37/KH5yamTJrA4kU/q1dZVCqVfFi7DhMnT1Ffd+Ofv/nn7+sAGBgaMv6bsRm+h1WpUixc5J4HTydE3jp+7CghISEAJMQnMMxlcIZlP/igJt9Pm/a2qiZekZ/xununAgRRcO4EBBD9arBWbGwsf1/PfH2KhMTUNRAUCgWTpkylSdOm/OHpya2bN3n58iVVqlalU+cu9OvfP93YhOuvrXvx7OlTnj19muF7lC1XTpdHEiLfvN6OHz58wMOHDzIsa2Ro+DaqJESeU6jScsL/EXHKlIKughA6k9+OhL4wMcyfnu7z/rp1VTe1074J13+JZBCEEELoHYmhdScBghBCCL0jWTbdSYAghBBC78hMBN29U9MchRBCCPFukAyCEEIIvSMJBN1JgCCEEELvyFLLupMAQQghhN6RDILuZAyCnnoaGsqcWTPp6NSGBnXtadWsCd+MGplugRdd/H39OvXrfEjHdm2zLvzKz/PnUe/DmgTeCciwzNkzp+ndozsN69nT0akNv65aSUqK9rUr7gUH42hfm18WzM9x/cX76+/r13Go/SEdnbLf9gBu37pF3Vo1M33t2rFd67VHDx9miLMzzRs3pkFde7p/+im/rlxJ0qsFw960d/duunbqSIO69nTt1JEdmWxAd+nCBerWqskfnp45eh4h8ptkEPRQYOAdhgwcQGRkJMbGxtjY2PDs2VNOnzrJ2TOncZ01m8969Mz1/RMTE5nlOi3DH9za/HX2DJs3bcy0jL+fH+PGjCY5OZlqtrY8CQlhmcdiXkRGMmHy9xrlly3xwMTEBGeXITl+BvF+SkxMZOb0nLW9NHcCUgPTUtbWlH9jd9E0Ja2s0v13SkoKs1yns2fXLgAqVa5MsWJFCQ4OYqnHYi6cP8fKNWvT7cNw5vQpXKdOwcjICJtq1bh/7x5uM2eQkBCvsaEZwJLF7pQrV47Pe/XK8TOJjMksBt1JgKCHpn8/mcjISBo2+oh5P/9CiRIlSE5OZsXSJaxZvYofZs+iYaOPMvyQzMqKpUsIDgrKdvkzp08xcfw4kpOTMy23bs1qlEolbnN/pEu3zwh7/pxe3bux9Y/NDB46jBIlSqjL+vn6cvzoEYYMH0Hx4sVz9Rzi/bM8h23vdQEB/gD0HzCQAc7O2brGc+Pv7Nm1iyKWlvyy2INGH30EpO4p8u3oMXhfvcr6dWsZOnyE+prVK1YAsGrtWho0bMTdwEC+6NmD1StW0Kfflxi+tvSy1/Hj+Ny4waw5P1BINnvKUxIf6E66GPTM3cBAfG/fRqFQ8MO8+eofqoaGhoz+5ls+rF2HpKQk/jx0MFf39719i40b1qfbXyEjiYmJLF/iwbejRxEfH59l+Rt//42pqRmdu3QFUjdrat3WCaVSyU2fG+nKLlm8CEtLS/oPGJir5xDvH9/bt9i4PnttT5vAVxkEW1vbbJWPjYlhxdKlAMxb+LM6OACwr1uPsePGAaizC5C6y+PtW7eoamNDg4ap20NXs7XFsWFDXrx4kS64SUlJYdkSD6pUrUoX2d4+zykUur2EBAh659mz1M2PihUrhrW15lriNWvVAiD0yZMc3zspKYkZU6cCClyGDc+iHs/o3qUzq1euwNDQkGkzZmV5/+fPwyhiWQQDg/9vlsVeZQf+fbWRFMB1b2/OnT3LwMEuFClSJMfPId4/SUlJuL5qe0OyaHsZuRNwB0j9gZ0dJ728iImJoUGjRjT7+GON823btWPkmDEMHDRY3eURGRlJcnIyRYsVS1e2uJZ2fOjAAQLv3GHk6DHpsgoibyh0/CMkQNA7pUuXAVI/qJ5q2SnxbmAgAGXKls3xvdf+upqAAH+cBw+mRo0amZaNCA/n8aNH2Nerx+ZtO/i8d+8s71/EsggvX7zg9f3DIiPCAShatKj62JLFi7CysqJPvy9z/Azi/bR29WoC/P1xdhlMdbvM2542ERERhIeHYW5unu22f+niRQBat2mj9XyRIkUYNmIkvfv0UQe1FhYWKBQKIiMi0r9/ePp2nJSUxIplS7H7oCbtP/kkx88jxNsgAYKesalWjbr1HABwnZI6FgFApVLx29o1XL/mjZmZGV265iylGXgngLWrV1HVxoYhr/W3ZqR4iRK4L1nG755/UD2LYCJN3XoOJCQksG/PHgDCw8I4dfIkhQoVonadOkDqYMdr3t64DBuOqalpjp5BvJ/u3Algzau2NzQbbU/rPfxTxx9Us7Xl1s2bzJs7l2GDBzF6+DBWLlvG8+eaO/8FBqZmHKraVCMhIYFdO7YzYdw4hg5yxm3mTG7887fGNWZmZtSoYcf9e/e4fOkSAMFBQVzz9qZEyZJUrlIFgN07d/Do4UNGfzNWBtPlE+li0J0MUtRDizyW8P2kCVy6cIGOTm2oVLkK4WFhhIeHYWNTjRlubpQuUybb90tOTmbGtGkolUpcZ7mlG7GdkdKlS1O6dOkc1dt58GD+OnOamdOnsun39TwJCSE6OhrnwS4UK1YclUrFMo/FlCtfnp6fy4jv/4LX296M2dlre9rceTW1NvDOHfr1Tt92zp45w4bf1vHj/AW0ei1bEBqS2g0XFxdLn549CQq6qz536eJFdm7fxtDhIxg5Zky6+w0eOpSJ341nxJAh2FSz4cH9+yQlJTFi1GgMDQ2Jj49n9cqV1HNwoEXLVrl6HpE1Cbx0JxkEPVSocGHs7etiYmJCfHw8Af5+hIeHAVDKulSOP2R/X/8bt2760LtPXxzq18+PKgOpGYQlK1ZSs1Yt7gUHU6RIEb4ZN56x48YDcOzoEXxv32bYiFHqEd8pKSk8DQ0lJiYm3+olCs7v63/jlo8Pvfvq1vYC/FMDhMTERIYMG8afx45z9e9/2L57D23btSM2NpYJ48fhe/uW+pqY2NQ2NWfmTF68fMH8n3/hwlVvTv51jrHfjsPAwIBVK5arM15pOnTsyE8LFlKpUiWC7gZRukwZZrrNoXefPgBs8fTk+bNnjP7mW/U1SqWS0CdPiIuLy/UzivQkg6A7ySDomaioKIY4D8Tfz5cmTZvxzfjvqFatGs+ePWPTxt/5Y9NGBn39NSvXrFF3RWTm/v17rFy2lDJlyqp/UOenJk2b0aRpM43jadM0q9rY8GnX1FkOXsePMddtNmFhYRgYGND1fGybAAAWxUlEQVSydWtmzp6jMUBMvJ/u37vHiqWpbe8bHdtew48aYWCgoHHTpnzSsZP6eA07O35Z7MGYkSM4c+oUHu7urFj9KwCJCQkAvHjxgg2enurvFzMzMwYPHUpcXCy/rlrFksXudO7SJd1Aw46dO9Oxc2eNekRHR/Pb2jU0btqUho1SZzls27IFD/dF/BsVhZFRIbp068r306ZjbGys0zP/18lAQ91JBkHPrF+3Fn8/X6rXqIHH8hV8ULMmhQoXpnyFCkz6fgoDnAcRFxfLj3PcsryXSqVi1vRpJCQkMMXVFXNz87fwBNrt37uH4KAgRrwa8f3o4UMmfvcdBoaGTJnuSr+v+nPKy4vx34zJ+mbinadSqZj5qu1NnaF72+vStRsz3eakCw5e5zJ0KACXL15S/xZvbJw6nbJx06Zag2lnlyEYGBjw7OlTfG/fzlY9Nvy2jhcvXjB67DcAXLt6lR9mz6JUqVK4zppFp087s3vnTmbPcM3xM4qCt2fPHuzs7DJ8LVq0KF15Hx8fhg8fTrNmzXBwcKBXr17s378/w/sHBwczfvx4WrZsSd26denSpQubNm3K1cJh2SEZBD1z4thRAAY4D6JQoUIa5wcPGcqm3zfg5+vLg/v3qVS5cob32rp5M9e8vfmkU+cC7StNSkxk1YrlfFCzJu3adwBSF7BRKpOYNecHdcYhNDSU40ePcOOfv7GvW6/A6it0t+VV2+vY+e20PbsPagKgVCbx/NkzKlWuTBHLIsTFxVKjhp3Wa9JmRIQ8fszjR4/UA2kzEhkZiefvv9O6bVvq2NsDsP63dQC4L1lK5SpV6NmrN3fv3uXQgQN8M/47rVOVRfYURDeBr68vAM2aNUu3sFuamjVrqv997tw5hg0bRkpKCg0bNsTU1JQLFy7wv//9j8DAQMa9WmcjjZ+fH19++SXR0dHUr1+fOnXqcOnSJdzc3Pj7779ZuHBhnj+PBAh65klICABVqtpoPW9ZtCjFixcnLCyMJ09CMg0Qjh07AsDhQwc5nMHCSk9CQqj3YWqjP3j0OOXLl9el+lpt37aVJyEhTJnuqh54lPYbW93XAoG6detx/OgRbt28KQHCe+740dS29+fBg/x5UHvbCwkJoW6t1LZ36FjWbS8xMZFChQppHbz2+tTatMC6SpWqPNMyVfh1adMbjQpl/VG6ZvUq4uLiGDV2rPqY7+3bFCtWTD27AcDevi63fHy4feuWBAg6MCiACOH2q8+lH3/8MdNB2vHx8UyYMAGAdevW0bhxYwAePHhA//79WblyJe3ataN27dpAavucOHEi0dHRzJ8/n26vFtaKiIhg4MCB7N+/n3bt2tGhQ4c8fR4JEPSMuYUFiRERhD1/rvV8QkKCeuqjublFpveqXr0GyUrtyyNHRb0k6O5dChcuTK0PUxuxcT4sFRsXG8va1aup51Cf5i1aqo+nDbp8fXlac4vUNHR0dHSe10O8XbbVa6DMYGnuqJevtb3aWbe9qJcv+cSpLTExMfy+ebPW7gJ/Pz8gdR0D61cf7LXr1OHypYvcunVT630TExPVC45VqFAx0+d5GhrK9i1b+KRTZ6pX//9pvxHh4RR/4zdNi1ftOEbasU4KIoPg5+eHlZVVljO49u7dS3h4OD169FAHBwCVKlXiu+++Y8KECWzcuJF58+YBqdkGf39/GjVqpA4OAEqUKMGMGTPo168fGzdulABBZK5Bw0YcO3KYPbt2ppuylebg/n0kJydjUaQIdnbaU6dpJk+dluG5M6dOMnbUSEpaWbF+U/7tQrdp4++Eh4cx7+ef0x0vVqwYDx884PnzZ5Qrl/qbY1pQJHszvP++n5Zx2zt96iRjR47EysqKDdloe5ZFi1LK2pqY4GD27dmrNUDYuGE9AG2cnNSDDTt07Mi6Nb9y7epVfG/fVq9CmmbHtm0olUoqVKyI3QcfZFqHlcuXkZyczIhRo9IdL1q0GJERkSQlJakzF89fteNi0o7fKw8fPiQqKoqWLVtmWfbs2bMAtG2ruSNpmzZtMDQ05MyZMxrlnZycNMo7OjpSsmRJvL29iY6OxsIi81/8ckIGKeoZ58EuGBkZceqkF4sWLiAuNlZ97tiRI+qtkQcNHpLut+/IyEiCg4J4+ODBW69zRqJevuT39b/RpGkz9br2aeo5pE552/bHH0DqEraHDx0CoL5jg7dbUfHOeP78GcFBQequtjQDBg0CYOf2bfzh6anuUkhISGDBvJ84fvQoZmZmDB81Wn3NBzVr0qFjR1JSUhg/dky6KZCXL11i+RIPAFyGDsu0Tg/u32ffnj10695Do0uvXn0HlMokdu/cAcDTp085feoUxsbG1MliTIPI3Nue5pg2/qBkyZK4ubnRrl076tSpQ4cOHVi2bBkJr2bFANy5k7oIl7YVaS0sLLC2tiYiIoKwsNRMaeCrFXAzWsG2atWqpKSkcPfuXa3nc0syCHqm1ocf4jrLjdkzXNnw2zq2b91C5SpVCA0NVS//2qXbZzi7uKS7bstmT1YtX0bZcuX489iJgqi6hvXr1vJvVBSjXo34fl3fr75i966drF+3llMnvYiMiODly5d82rUrNtWqFUBtxbvAY9Ei9u3ZQ4OGDVm74Xf18R49P+eWz012bNvKTz/MYdXyZZQrX5779+4RHR2NiakpizyWaIxjcJ01m6dPn/L3tWv0+fxzbGyqoUKl3nSpZ69edO+Z+dbpy5Z4YGhoyLARmqtADnIZwumTp5jr5saWzZsJffKEmJgYho4YgeVry4uLnHvb0xzTxh/s2rWLokWL4ujoSOnSpbl58yYeHh6cPXuW9a82G0vLEpUqVUrrvUqVKsWTJ08ICwvDysqKZ8+eZVkeUAcUeUUCBD3U9bPPsPvgA35fv44rly9zJ+AOZmZmfNSkCT179aZ9h3d/7ffwsDA2e26ijZOT1tHh5cqVZ81vG1g47ydu/PM3lpaWOA92YeSYsVruJgRMnzmTJs2asn3LFm7dukWAvz+lSpWiQ8eODBoylApatj+3sLBg7foNbNuyhQP79hJ0NwijQkbUd3Skd9++dOykudbB6wL8/Tny5598+fXXWlcvrV2nDit+/RX3nxeq6zNsxEh1xkPknq5jEKKiooiKitI4bmlpiaWlpcbxtAxCx44dmTt3LmZmZgA8evSIUaNGcf36ddzd3Zk8ebJ6Km1GO5OmHY99lQHOafm8olC9Pnz3PyBOmT/zRYV4m2SlN6EvTAzzp6c78Om/WRfKxJ/b1rP01Xbfrxs9ejRjxmiut5KQkMDDhw+pVKmSxmq1vr6+dO/eHVNTUy5fvoyDgwNKpRK/V4Nj3/TVV19x5coVNm/ejKOjI+3bt+f+/ft4eXlpna0zadIk9uzZw4IFC+j6aiG5vCAZBCGEEOINAwYMoHv37hrHtWUPAIyNjbHNYCvxmjVrUqZMGZ48ecK9e/cwNTUlKiqKhIQErStmpo1XSMtCpP0dHx+v9f5vls8rEiAIIYTQO7pm2TLqSsgtKysrnrzab8Pa2pqoqCieP3+utWvrzTEK1tbW+Pr6EhYWRjUtY6yyGtOQWzKLQQghhN5RKBQ6vXIiOjqa6dOnM3bsWJRKpdYyjx49AlJ3uq1evTqA1lkH0dHRPHv2jBIlSmBlZQWgLp82m+F1KpWKoKAgDA0NtQYPupAAQQghhN5R6PjKCXNzc44dO8aRI0e4cuWKxvkzZ84QGRlJjRo1KF26NM2bNwfg+PHjGmW9vLxITk5Ot55CWvkTJzRnmF27do2IiAgcHR3zdA0EkABBCCGEHnqbGQSFQkHv3r0BcHNz4+lrS3Q/ePCAWbNmATDi1VTXDh06ULJkSXbv3s3p06fVZR8+fMjPP/+MQqFg4MCB6uONGjWievXqnDt3jm3btqmPR0REqO/t7Oycsy9Qdp5LZjEI8f6RWQxCX+TXLIZ7YTE6XV/FKmc7iMbHxzNo0CC8vb0xMzPD0dERgEuXLpGYmIizszOTJ09Wlz9x4gRjx44lOTmZhg0bYm5uzsWLF4mLi2PcuHEMHz483f1v3LjBgAEDiI2NpW7dulhbW3P58mVevnxJ7969cXPLeofenJIAQYj3kAQIQl/kV4BwP1y3AKFyyZxvMZ6YmMj69evZv38/9+7dS90vpFYt+vfvT/v27TXKX7t2jWXLlvHPP/+gUqmwtbVl4MCBdOzYUev9AwMD8fDwUAcdlStXpk+fPvTq1Uu9RHhekgBBiPeQBAhCX+RXgPBAxwChUi4CBH0j0xyFEELoH4midSaDFIUQQgihQTIIQggh9I7kD3QnAYIQQgi9Iz0MupMAQQghhN6R+EB3EiAIIYTQP5JC0JkMUhRCCCGEBskgCCGE0DuSP9CdBAhCCCH0jvQw6E4CBCGEEHpIIgRdSYAghBBC70gGQXcySFEIIYQQGiSDIIQQQu9IAkF3EiAIIYTQO9LFoDsJEIQQQughiRB0JWMQhBBCCKFBMghCCCH0jnQx6E4CBCGEEHpH4gPdSYAghBBC/0iEoDMJEIQQQugdhUQIOpNBikIIIYTQIBkEIYQQekcGKepOAgQhhBB6R+ID3UmAIIQQQv9ICkFnMgZBCCGEEBokgyCEEELvSP5AdxIgCCGE0DvSw6A7CRCEEELoHYkPdCcBghBCCP0jKQSdySBFIYQQQmiQDIIQQgi9I/kD3UmAIIQQQu9ID4PuJEAQQgihhyRC0JUECEIIIfSOZBB0J4MUhRBCCKFBMghCCCH0jiQQdCcBghBCCL0jXQy6kwBBCCGEHpIIQVcyBkEIIYQQGhQqlUpV0JUQQgghxLtFMghCCCGE0CABghBCCCE0SIAghBBCCA0SIAghhBBCgwQIQgghhNAgAYIQQgghNEiAIIQQQggNEiAIIYQQQoMECEIIIYTQIAGCyDPnz5/n66+/5qOPPqJ+/fr079+fs2fPFnS1hMi1Xbt2YWdnx9WrVwu6KkK8dRIgiDyxa9cunJ2duX79Ovb29jg4OHD9+nVcXFzYunVrQVdPiBy7fv06bm5uBV0NIQqM7MUgdPbs2TPatm2LsbExmzdvpkaNGgDcuHEDZ2dnkpKSOHbsGKVLly7gmgqRPUePHmXy5MnExMQA4OnpSYMGDQq4VkK8XZJBEDrbtGkTiYmJDBw4UB0cANjb2+Pi4kJCQoJkEcR7ITQ0lIkTJzJmzBhSUlKwsrIq6CoJUWAkQBA6Sxtn4OTkpHGuXbt2AJw5c+at1kmI3HB3d2fv3r3Url2brVu3YmNjU9BVEqLAGBV0BcT7TaVSERgYiIGBgdYP0ypVqmBgYEBgYCAqlQqFQlEAtRQie2xsbJg3bx5du3bFwEB+fxL/bRIgCJ28fPmSxMRESpQoQeHChTXOGxkZUbx4ccLDw4mJicHCwqIAailE9gwdOrSgqyDEO0NCZKGTuLg4AExNTTMsY2JiAqAe8CWEEOLdJwGC0El20rAyUUYIId4/EiAInZiZmQGQkJCQYZm0c5llGYQQQrxbJEAQOrGwsMDMzIzIyEiUSqXGeaVSSWRkJMbGxlhaWhZADYUQQuSGBAhCJwqFAltbW5KTk7l3757G+eDgYFJSUtKtjyCEEOLdJwGC0Fnz5s0BOH78uMa5tGMtW7Z8q3USQgihGwkQhM569OiBsbExv/76Kzdv3lQf9/HxYc2aNZiYmNCvX78CrKEQQoicknUQhM4qVKjApEmTmD17Nn369KFx48aoVCouXbqEUqlk3rx5lCxZsqCrKYQQIgckQBB54ssvv6RcuXKsWbMGb29vChcuTP369RkxYgRNmjQp6OoJIYTIIdnNUQghhBAaZAyCEEIIITRIgCCEEEIIDRIgCCGEEEKDBAhCCCGE0CABghBCCCE0SIAghBBCCA0SIAghhBBCgwQIQuQjFxcX7OzsGDp0qM73+vfff9m0aVMe1CprkydPxs7ODl9f37fyfkKId48ECELkk+fPn3P+/HlMTU3566+/CA0N1el+HTp0YPv27XlUOyGEyJwECELkk/3795OcnIyLiwvJycns3LlTp/uFh4fnUc2EECJrEiAIkU/27NlD0aJFcXFxoUiRIuzcuRNZ2VwI8b6QAEGIfODn54e/vz9NmjTBxMQEJycnHj9+zPnz57WW37lzJ7169cLBwYFmzZoxcuRI/Pz8ALh06RJ2dnbq+9rZ2bFkyRIA2rRpQ4MGDTTul3bNDz/8kO54QEAAEyZMoGXLltSuXZv69evTp08fjhw5kpePL4TQAxIgCJEP9uzZA0CnTp3S/a1tDIGrqytTpkwhPDycbt260apVK86dO0ffvn3x8/OjfPnyjB49GgArKytGjx5No0aNclynGzdu0KtXL06dOsXHH3+Ms7MzH3/8MT4+PowdO5aTJ0/m9nGFEHpItnsWIo8lJydz4MABzM3NadWqFQDNmjWjZMmSHD9+nIiICEqUKAHAhQsX2Lp1Kw0aNGDVqlVYWFgA0LNnT/r164e7uzsrV65kzJgxLF26FCsrK8aMGZOrei1evBilUsmuXbuoVq2a+vihQ4cYN24cBw4coHXr1ro9vBBCb0gGQYg8du7cOZ4/f067du0wNjYGwNDQkE8++YSkpCT27dunLnvw4EEAvvvuO3VwAFC/fn3Gjx+fpz+wBw4cyIIFC9IFBwAfffQRIIMghRDpSQZBiDy2d+9eADp37pzueJcuXfD09GT79u0MHDgQSB1TYGhoSJ06dTTukxdrJ7yuefPmQOr0Sz8/Px48eEBwcDDe3t5AauZDCCHSSIAgRB6Kjo7m+PHjAAwZMkRrmcDAQK5fv46DgwNRUVEYGxtTqFChfK/bkydPcHNzw8vLC5VKhYGBAVWqVMHR0ZHbt2/n+/sLId4vEiAIkYcOHz5MfHw8derUoVatWhrng4ODuXz5Mtu3b8fBwQEzMzMSEhJQKpUYGaX/doyLi8PU1DTL99Q2dTI+Pl6jzNChQwkMDGTYsGE4OTlRvXp1TExMCAsLkwWYhBAaJEAQIg+ldS9MnjxZ6/TDkJAQ2rZty59//smUKVOoUaMGvr6+3L59G3t7+3RlR44ciY+PD2fPns0wUChUqBDx8fGoVCoUCoX6+IMHD9KV8/f3JyAggA4dOjBu3Lh05+7evQtoDzSEEP9dMkhRiDzy+PFjrly5Qvny5XF0dNRaply5cjRu3JjY2FgOHTpE165dAXB3d0/3W//169e5fPkyDg4O6uCgUKFCJCUlpbufjY0NSqWSM2fOqI+9ePECT0/PdOUKFy4MaA5EfPHiBfPnzwdAqVTm5rGFEHpKMghC5JG9e/eiUqno0qVLut/m39SjRw/Onz/P9u3b2b59Oz179mTnzp1069aN5s2bExMTw8GDBzE3N8fV1VV9nbW1NUFBQcyYMYOWLVvSpk0bevfujZeXF99++y1dunShcOHCHD58mMqVKxMcHKy+tkqVKtjb23P16lX69etH/fr1iYyM5Pjx4yQmJmJqakpkZGS+fn2EEO8XySAIkUfSpi+mZQUy0r59eywtLblx4wb+/v788MMPzJgxAxMTE7Zu3cqxY8do0aIFf/zxBxUrVlRf5+rqSoUKFdi5cycnTpwAoHXr1ixcuJBKlSqxe/dujh07Rvfu3XF3d0/3ngYGBixfvpwePXrw6NEjNm7cyNWrV2nRogU7d+6kWbNm3Lt3T6NrQgjx36VQScejEEIIId4gGQQhhBBCaJAAQQghhBAaJEAQQgghhAYJEIQQQgihQQIEIYQQQmiQAEEIIYQQGiRAEEIIIYQGCRCEEEIIoUECBCGEEEJokABBCCGEEBr+D6K6vq21MLoMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cf_matrix = metrics.confusion_matrix(test_labels,b2_pred)\n",
    "TN = cf_matrix[0][0]\n",
    "FN = cf_matrix[1][0]\n",
    "TP = cf_matrix[1][1]\n",
    "FP = cf_matrix[0][1]\n",
    "fl = [TN, FN, FP, TP]\n",
    "fig, ax = plt.subplots(figsize=(8,6))\n",
    "\n",
    "sns.set(font_scale=2)\n",
    "group_names = ['TN','FN','FP','TP']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in fl]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in\n",
    "                     fl/np.sum(cf_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "\n",
    "sns.heatmap(cf_matrix, annot=labels, fmt='', cmap=\"BuPu\")\n",
    "plt.ylabel('Predicted', fontsize=20)\n",
    "plt.xlabel('Actual', fontsize=20)\n",
    "ax.set_ylim([0,2])\n",
    "ax.invert_yaxis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XI6cWC33Zz-f",
    "outputId": "69561842-0677-4ad4-b25c-27d48713e275"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWIAAAFgCAYAAACBlHNxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hc1Z3/8fd31Hu3ZUtyw4Ab2NiyccXdIg2yS7JJNgWySUjZZENLAiGhs0AgmGzKBpJs+v6yqUuySZAr7jauGNtgbOMi2bItq3dpZs7vjzuSZsZjaSTN6I4039fzzCPp3HtnzvVIH58599xzxBiDUkop+zjsroBSSkU7DWKllLKZBrFSStlMg1gppWymQayUUjbTIFZKKZtpECullM00iFVYiIjxe7hEpFpEXhWRO0REejl+hYj8j4icEZFWEakRkd0i8rCIZPVyrENEPiAifxCRMs/xTSLypoi8JCIL+nE+k0TkuyJySETqRKRdRM6JyF9F5FMiktjX51Sqk+gNHSocRKTzF+tRz9c4YCLwD57vv2+M+WKA4xKAHwMfA1qAvwNvA6nAMmAKcAm4zRizOcDx+cDvgQVAA7AWOAEIcDWw3PNc/2aM+W6Q5/IQ8DBWw2UnsBtoBEYCS4AJwF5jTHEwz6eUPw1iFRadQWyMEb/yBcBmrGC8yhhz0m/7fwGfBPYB7zfGlHltE+Bfge8AzcAcY8ybXtuTge3AdOA3wBeMMTV+z58O3Ae0GWOeDOI8vg48CZQBHzTG7Aqwz3uBe40xS3t7PqUCMsboQx8hfwDG+vUKuO2wZ/sH/MoXesqrgVE9PPfTnv3W+pU/6CnfCjh6qV9CEOcwDmj3PKYF+3xYrWQDPHKFfU8Bp/zK7vAccwdwM/AqUOcpKwBcwL4eXv8Vz77T/MpvxPqEcN5zHmXAi8Bou39H9NH90D5iZYfOVnKHX/lnPF9/ZIyp6OH4Z4A2YIWIjPcqv9Pz9XFjjLunChhj2oKo5yexulH+YIw5FILnC8YHgP/D6lb5IfBbY8xZYB1wg4hc53+AiIwCVmB1jxzyKv8ksA14F7AReAHYA3wa2CMiY0JUZzVAsXZXQEUXEbkJuBardfaa3+aFnq/renoOY0yNiOwF5mP1BZ8UkSJgDOAENoWoup31WR+i5wvGu4F3G2Ne8Sv/GbAKuB2ra8Xbx4AY4OedBSJyDVbL9xSw2BPmnduWYfWdfwerz17ZTINYhZWIPOL51vtinQD3BWj1jvJ8LaN3nfuM9ju2yhjT2r/aXqbzOctD9HzBeDlACAP8L1ZXxUdF5GvGGJfXttuxPl38P6+yz2P9m3/ZO4QBjDEbROTPwPtEJM0Y0xDaU1B9pUGswu1hv58N8CljzE97OCaYK8id3RvmCj+HQjieszf+nxKsChjTIiK/xeq+KQH+BiAis4CpwJ+MMZe8Dpnn+bpYRGYHeMoRWK3oa4C9Iaq76icNYhVWxjNqQkRSsMLhJ8APReS0MWaD3+7ngfFYXQxHe3nqQs/Xzlb1Oc/XXBFJDFGr+Bwwyeu1BsP5Hrb9DCuIb8cTxJ7vwatbwiPH8/Urvbxeal8qp8JDL9apQWGMaTLGrAPeh6c/0zPczNtWz9cVPT2X54aOWZ4ft3mevww4g9W4uClE1e6sz/I+Htd5ofBKDZ2MHo69YuvbGLMdOAbcKiKZIhIHfARrXPXf/Hav63wtY4z08AhVf7oaAA1iNaiMMQeBH2G1Mu/22/xjz9dPi8jIHp7mPiABWGd8xyG/5Pn6DRHp8Xfbc+NIb36K1fd6m4hM6cPzdY5dLgqw30QgM4jXvpKfY537h4D3ALnAfxtj/Eeg7PR8XTSA11KDxe7xc/oYng96HkdcgHXXXA2Q5bftF55jdwOFAY79HNbIiAZgit+2ZOCA5/hfAZkBjk8FHgIeDPI8vu55vpNA8RX2uRnY4PVzHFaLtBYY4VWehNVyNfQwjriX+hRhjSneBvzRc8wNAfabhDUy5W3gmgDb44FFdv+e6MN6aB+xGnTGmLMi8iLwZeCrwANem+/E+kj/EeCoiPwd6+N4CrAUmAZUYd3ifMTveZtF5GasGxg+ijUqYC1wHOvT30SsboZ04LLbq69Q138XkVisi467RWQ71ljczlucb8K6dXqP1zEdIvId4JvAfhH5k+ecVmL1O5+jn4wxZSKy0XMeTuANY8z+APu9JSL/AvwXcFhEXsEK5TisPvhFQCVWYCu72f0/gT6G54MeWsSe7SOBJs9jZIDtq4DfYQ0da8NqYe4FHgGye3ltB/BBrBZjOdCKdUv0W1jdH/P7cT6Tge8Ch4B6rNZmBdZcGJ/C7049rBEX92PNc9GO1X/9LaxW+yn62SL27Puxzn9frFure9r3OqyLfKc9/47VnnN4EVhm9++JPqyHzjWhlFI204t1SillMw1ipZSymQaxUkrZTINYKaVspkGslFI2i+pxxLm5uWbcuHF2V0MpNczs3bv3kjEmL9j9ozqIx40bx549e3rfUSml+kBETvdlf+2aUEopm2kQK6WUzTSIlVLKZhrESillMw1ipZSyme1BLCKFIvJdEdkhIs0iYkRkXJDHOkTkARE5JSKtIvK6iNwW3horpVRo2R7EWHPE/hPWJOFb+njs41jTIn4PeBfWqgS/E5F3h7KCSikVTpEwjnizMWYkgIh8Gmse2l6JyAisJXOeNsY85yne6FmK5mkuX8NLKaUiku0tYmOMu/e9AirBWu7lV37lvwKuE5HxA6qYUirq1Z2pY8fqHWF/nUhoEffXVKwVB477lR/2fJ2Ctc6YUkr1SXtjOxt/uZ/Dv3iDhp1nyZ+ez/hl4Wvb2d4iHoBsoNZcvsRItdf2y4jInSKyR0T2VFZWhrWCSqmhxbgNB352gJ988k80LCyg6HsrSbo+j9K7SzHu8K1mNJRbxIK1Zleg8isyxryEZ9n14uJiXSdKKQXA6S2nKb27lKbcRMY8txyJc0B8DOOeWcac5CTE0WO0DMhQDuJqIEtExK9VnOW1XSmlelRzsoZ1X13Hkd8fAYcw8Xf/YIUwENvipGTpRFISwhuVQ7lr4jCQAFzlVz7F8/UISil1BW31bax7YB3fn/R9K4QB3IayL67BUdtGamwMK6eNDnsIw9BuEb+CtUz5R4FHvco/BhwyxuiFOqXUZdwuNwd+eoAN39hA04Umn23TPjKNFU+vIH50KgIkxsYMSp0iIohF5AOeb2d5vr5LRCqBSmPMJs8+TuDnxphPARhjLorIauABEWkA9gEfApYBtw7qCSilhoSTG09SencpF16/0FUWPyadvPw0SlaXUDS/yJZ6RUQQA7/z+/kHnq+bgCWe72M8D28PAo3Al4F84CjwT8aYv4Snmkqpoaj6eDVrv7KWt/73LZ/ygrtmk33nDOYUZFGYnmRT7SIkiI0xvV6ODLSPMcYFPOF5KKWUj9a6VjY/sZld39mFu6P73rHYpFhm/ux9tEy3VjPaXVFLQqyDvOQEW+oZEUGslFKh5Ha62ffjfWx8aCPNlc0+2677+PVMeGQR73R0dJXlJMWTmRg32NXsokGslBpWTqw9wZp71nDx0EWf8sJ5hZS8cDO149N4u7r7Il1ecjzzCrKIddg3iEyDWCk1LFw6eom1963l7f9726c8Y0wGK761gikfnMKhS40c9wrhEckJzCvIIiaMN2sEQ4NYKTWktdS0sOmxTez+3m7czu5+4LiUOBY+sJB598wjNjGW1y/W805tdzdFfkoCN462P4RBg1gpNUS5OlzsfXEvrz78Ki3VLd0bBGbcMYNlTywjbXQaxhj2X6jnVF13CI9OTWTO6EwcYn8IgwaxUmoIOv7KcUrvKeXSm5d8yscsGkPJ6hJGzxrdVVbV0u4TwoVpiRSPipwQBg1ipdQQUnmkkjX3ruH4K76z32aOy2TlcyuZ/I+TEb+AzU1OYMbIdA5cqGdMehIz8zMiKoRBg1gpNQQ0VzXz6iOvsuc/92Bc3XN8xafFs+jBRcz98lxiE68cZxMyU0iNiyUvOf6yoI4EGsRKqYjlanex+we72fToJlprW7s3CMz89EyWPr6U1JGpvse4DW5jiIvxHY42IsWemzWCoUGslIo4xhiO/fUYa+5dQ9XbVT7bxi0dR8nzJeTPyL/sOJfbsPNcDR0uNwuKsomzcWxwX2gQK6UiyoU3LrDmnjW8s+4dn/Ksq7JY9e1VXHvLtQG7F5xuw86z1Vxsbgdge3kNCwuzI2J4Wm80iJVSEaGpsomND21k30v7fJYlSkhP4KaHbmLOF+cQe4W5gZ1uN9vLa7jU0t5VlpsczxDIYECDWCllM2ebk9e++xqbH99MW31bV7k4hFmfncWSR5eQkpdyxeM7XG62n62mqqV77ojJOalMzk0La71DSYNYKWULYwxHXz7KmvvWUHOixmfbhJUTKHm+hBHTRvT4HO0uN9vKq6lp7Q7hablpXJOT2sNRkUeDWCk16M4fOE/p3aWcevWUT3nONTmsen4VV7/76l6HmbW73Gwtq6K2zdlVdn1eOhOzr9x6jlQaxEqpQdN4vpEN39zA/p/s91mDPTErkcUPL2b2F2YTE9f78kRtThdby6up8wrhGSPSmZA19EIYNIiVUoPA2epk5ws72fLkFtobuy+oSYww+wuzWfzwYpJzkoN+voMX631CeObIDMZlBn98pNEgVkqFjTGGN//wJmu/spbaU7U+2ya+ayKrvr2KvMl5fX7e6SMzqG93UtfmZFZ+BmMzhm4IgwaxUipMzu09R+ndpZzZcsanPHdyLiXPlzDx5on9fu74GAcLC7Opau1gdGriQKtqOw1ipVRINZxrYMODGzjw8wM+/cBJOUkseXQJxZ8txhHbtzveXG5z2Y0ZCbExjE4dnOXuw02DWCkVEh0tHez49g62Pr2Vjqbu4WSOWAdzvjSHm755E0lZfV8pubHdydayaqbmpVFk40rL4aRBrJQaEGMMh35ziHVfW0d9Wb3PtmtvuZaVz64k55qcfj13Q5uTLWVVtLrc7KmoxSFQkDb8wliDWCnVb+W7yim9u5TyHeU+5SOmjaBkdQkTVkzo93PXtXWwtayaNpe1/JFDGDKT+PSVBrFSqs/qy+tZ/8B6Dv7qoE95cl4ySx9fysxPzexzP7C32tYOtpZX0e6ZezhGhPmFWeQlR+5UlgOhQayUClp7Uzvbn93Otm9tw9nSPY7XEedg7l1zWfTgIhIzBjaKoaa1na1l1XR4Jv6JdQgLCrLJSY4f0PNGMg1ipVSvjNtw8NcHWf/AehrONvhsm/yPk1nxrRVkX5U94NepamlnW3k1Tk8IxzmEBYXZZCcN3xAGDWKlVC/Ktpfxyl2vcG73OZ/y/Bn5lKwuYdyScSF5nUvNbWwvr8FprBCOdwgLinLISowLyfNHMg1ipVRAtadrWX//eg795pBPecrIFJY9uYwZd8zAEROai2dtTjfbz3aHcILnho2MKAhh0CBWSvlpb2xn69Nb2fHtHThbu/uBYxJimHfPPBY+sJCEtNBeNEuIdTB9RDp7z9eREONgUVE26QnREcKgQayU8jBuw4GfH2DD1zfQeL7RZ9vUf5rKimdWkDkuM2yvPzYjGQGykuJJi4+uaIqus1VKBXR682lK7y6lYl+FT/moWaO4+YWbGbNwTMhf0xhz2ZzDY4b45D39pUGsVBSrOVnDuq+u48jvj/iUp45KZflTy5n+8elIGBZ+K69v4VhNEwsKs4kPUT/zUKZBrFQUaqtvY8u/b2Hn6p242l1d5bGJscz/ynwWfHUB8anhGTJ2pq6ZPefrANhWXs3CwmziojyMNYiViiJul5v9/7Wfjd/YSNPFJp9t1/3zdSx/ajkZYzLC9vqn6prZ5wlhAKfb4DKG6LksF5gGsVJR4uSGk5TeXcqFgxd8ygtuLKBkdQlF84rC+vrv1DZx4EL3pEDp8bEsLMomMXZ4TGU5EBrESg1zVceqWPuVtRx9+ahPeXphOiueWcG0D08LSz+wtxM1Tbx+sTuEMxJiWViUQ0KUd0l0sj2IRaQIWA2sBARYB9xljDnT44HWsWOAx4GlQC5QDvwWeMoY09TTsUoNd621rWx+YjO7/mMX7g53V3lcchwLvraA+ffNJy45/J0Cb1c3cqiy+7borMQ4vUjnx9YgFpFkYAPQBtyONZ//E8BGEbm+pzAVkRSs0I4DvgmcAWYDjwJXAx8Kb+2Vikxup5u9P9rLqw+9SvOlZp9t0z8xnWX/voz0gvRBqctbVQ0cudQ9JjnbE8LRfnHOn90t4s8AE4BrjTHHAUTkIHAM+CzwfA/HLsAK3BJjzBpP2UYRyQbuE5FkY0zzlQ9Xavg5seYEpfeUUnm40qe8aH4RJS+UUDC7YNDqcqq22SeEc5PimVeYNWznFB4Iu4P4FmBnZwgDGGNOisg24FZ6DuLOsTX1fuW1gAOrm0OpqHDp6CXW3LuGY3895lOeMTaDld9ayZQPTrns5olwK0hL5J26ZmpbO8hLjmdeQRaxGsIB2R3EU4GXA5QfBj7Yy7HrsFrOz4jI57G6JuYAXwZ+qH3EKhq0VLew6bFN7P7+btxOr37glDgWfX0Rc++eS1ySPYPD4jwT97xV1cjU3LTLFv9U3ewO4mygJkB5NZDV04HGmFYRWQj8ASu4O/0Y+GLIaqhUBHJ1uNjzwz1semQTLdUt3RsEZnxyBsueWEbaqDT7KugRH+Pg+hGD0x89lNkdxOCz4HaXXv/rFJFE4H+AEcDH6W4RPwQ4gc9f4bg7gTsBxowJ/f3zSoXbsb8fY809a7j01iWf8rE3jaVkdQmjZo4a9DoZY9h/oZ7sxDjGZUbnfBEDYXcQ12C1iv1lEbil7O1TwBJgojHmhKdss4jUAS+JyA+NMa/7H2SMeQl4CaC4uDjQfwJKRaSLhy+y5t41nCg94VOeOT6TVc+tYtI/TBr0fmCwQnjv+TrO1Ldwqs5a5DNaJ+/pL7uD+DBWP7G/KcCRAOXergNqvEK402uer5OBy4JYqaGm+VIzrz7yKnt+uAfj6m47xKfFc9M3buLGL99IbII9f8puY9hTUUt5Q2tXWWVzuwZxH9kdxH8GnhORCcaYdwBEZBzW0LT7ezn2PJAlIhO9R10AN3q+ng1xXZUaVK52F699/zU2P7aZ1truoBOHcMOnb2DpY0tJHZlqW/3cxvDauVrONXbXbVxGEjeMDN9cFcOVGGPfp3PPTRmvAy3AN7D6ix8H0oDrjTGNnv3GAieAx4wxj3nKxgEHsQL5Saw+4mKsmzveBuYYY9z0oLi42OzZsyfk56XUQBhjePv/3mbNvWuoPlbts238svGsen4V+dPzbaqdxeU2vFZRQ0VjW1fZhMxkpo9It6V7JNKIyF5jTHGw+9vaIjbGNInIMqxbnH+JdZFuPdYtzt5LBAgQgzU+uPPYUyIyF3gE6268XKAMq//3yd5CWKlIdOHgBUrvKeXk+pM+5dkTs1n17VVc875rbA86l9uw81wNF5q6Q3hiVgrX5aXZXrehyu6uCTxzStzWyz6nCDCSwhhzBPin8NRMqcHTdLGJjQ9tZN+P9mHc3Z9SEzISWPzwYub86xxi4u2fpczpNuw8W83F5vausmuyU5iaqyE8ELYHsVLRzNnmZNd/7GLLE1toq+9uYYpDmPW5WSx9dCnJuZFx4cuYy0N4Uk4qk3NSNYQHSINYKRsYY3jrf99i7X1rqXnHd6TmVauuYtW3VzFi2gibaheYiDAuI7kriKfkpjIpx/6bRoYDDWKlBlnF/gpK7y7l9KbTPuU51+ZQ8nwJE981MWJbmIXpSbiNodXl5pps+0ZsDDcaxEoNksbzjax/cD0HfnrA537SxKxEljy6hOLPFRMTZ38/cG90jHDoaRArFWbOVic7Vu9g679vpb2xu3/VEeug+AvFLHl4CUnZSTbWMLA2p4vXL9YzfUQGCbE6a1o4aRArFSbGGI78/gjrvrqO2lO1Ptuufs/VrHpuFbmTcm2qXc9anS62lFXT0O6kod3JoqIcXVEjjDSIlQqDc3vOUXp3KWe2+q74lTclj1XPr2JiyUSbata7lg4XW8qqaOxwAVDX5qSqpZ1RqYk212z40iBWKoTqz9az4esbeP0XvtOcJOUksfTxpcz6zCwcEfwxv9kTwk2eEBZg9qhMDeEw0yBWKgQ6mjvY/u3tbHt6Gx3NHV3ljjgHc740h8XfXExiZmSHWVO7ky1l1TQ7u0N4zuhMCtIir/96uNEgVmoAjDEc+s0h1n1tHfVlvqt2XXvrtax8diU5V+fYVLvgNbY72VJWRYtnlQ+HwI2js7QlPEg0iJXqp/Kd5ZTeXUr5znKf8hHXjaBkdQkTlk+wqWZ9U9/WwZayatpc3SE8ryCbkSkJNtcsemgQK9VHdWV1rL9/PW/89xs+5cl5ySx7Yhk3fOoGHENkhEGDpzuiM4RjPCE8QkN4UGkQKxWk9qZ2tn1rG9uf3Y6zxdlVHhMfw4133ciiry8iMWNofZRPinWQGh9DW4ubWBHmF2aRm6whPNg0iJXqhXEbDv76IOvvX0/DuQafbZNvm8zKb60ka0KPa91GrFiHg/kF2ew6V8Pk3DRykuLtrlJU0iBWqgdntp2h9K5Szu0551Oef0M+JatLGLd4nD0VC6G4GAcLiyL/guJwpkGsVAC1p2pZ97V1HP7tYZ/y1PxUlj25jOm3Tx8y/cDeLjW3Ud/uZEJmit1VUV40iJXy0tbQxtant7Lj2ztwtbm6ymMSYph37zwW3r+QhLSh2Yd6samNHWercRkQhPG67H3E0CBWCqsf+MDPD7Dh6xtoPN/os23qh6ay4ukVZI7LtKl2A3e+qZWdZ2voXPzjzaoGitITiXUMvVb9cKRBrKLeqU2nKL27lPP7z/uUj549mpLVJYxZMMammoVGRWMru851h3BSrINFRTkawhFEg1hFrZp3alj71bW8+Yc3fcrTRqex/OnlXP/R6xFHZE7QHqyzDS28dq62a/rj5NgYFhVlkxKvf/qRRN8NFXXa6tvY/ORmdr2wC1d7dz9wbFIs878ynwVfXUB8ytAfxlVe38Luiu4QTomLYVFRDslDYPL5aKNBrKKG2+Vm/0/2s+EbG2iubPbZdt1Hr2P5U8vJKMqwqXahdaaumT3n67p+TvWEcJKGcETSIFZR4eSGk5TeXcqFgxd8ygvnFlLyQgmFNxbaVLPQO1XXzD6vEE6Lj2VRUTaJsRrCkUqDWA1rVceqWPuVtRx9+ahPeXpROiueWcG0D0+L2IU6+yve4UCwlsXLSIhlYWE2CRrCEU2DWA1LrbWtbHp8E6999zXcHe6u8rjkOBY+sJB598wjLjnOxhqGz+i0RGaPzuRYdRPzC7NJGII3nkQbDWI1rLidbva+tJeND22kparFZ9v026ez7MllpBek21S7wVOYlkRBauKwa+0PVxrEatg4seYEpXeXUnmk0qd8zMIxlKwuYXTxaJtqFl4napooSEu8rA9YQ3jo0CBWQ96lty6x5r41HPvrMZ/yjLEZrHx2JVM+MGVYhpIxhiNVjRytauRkbTOLirQveKjSIFZDVkt1C68++ip7frAHt7O7Hzg+NZ5FDy5i7l1ziU0cnr/ixhgOVTZwrKYJgPp2J29VNTJ95PAYfhdthudvqRrWXB0u9vznHl595FVaa1q7Nwjc8C83sOyJZaTmp9pXwTAzxnCwsp4TNd1joUemJDAtb/j3fQ9X/Q5iEckCUo0xZSGsj1JXZIzh+N+PU3pPKVVHq3y2jV08lpLVJYy6YZRNtRscxhgOXKznZG13CI9KTWDOqCxihvjt2NGsT0EsIqnAo8BHgTysoYqxnm03Ag8D3zDG7AtxPVWUu3j4ImvuXcOJ0hM+5VkTslj53EomvX/SsOwH9maMYd+FOk7XdY8GKUhLZPaoTBzD/NyHu6CDWEQygK3AVOAAcAmY7LXLG8Ai4COABrEKieZLzWx8eCN7X9yLcZmu8oT0BG765k3M+dIcYhOGfw+b2xj2na/jTH13CBelJTJLQ3hY6Mtv8INYIXyHMeYXIvIw8FDnRmNMs4hsApaHuI4qCrnaXbz2vdfY9Ngm2urausrFIcz8zEyWPraUlBHRscqE2xj2VNRS3tDdHz4mPYlZ+RnD/lNAtOhLEP8jUGqM+UUP+5wGZg+sSiqaGWM4+uejrL1vLdXHq322jV8+npLVJYy8bqRNtbOHMXQtdw8wLiOZG0amawgPI30J4kLgD73s0wjo+BnVLxcOXqD07lJObjjpU559dTarvr2Ka957TVSGT4xDmFeQxbbyGjISYpk+QkN4uOlLEDcAI3rZZzxW33HQRKQIWA2sBARYB9xljDkT5PGTgceApUAKcAb4gTHmO32ph7JP08UmNnxzA/t/vB/j7u4HTsxMZPHDi5n9hdnExEf3jQqxDgcLC7NxiN4xNxz1JYh3A+8VkTRjTIP/RhEZBbwb+L9gn1BEkoENQBtwO9YojCeAjSJyvTGmqZfjiz3Hvwp8GqgDrgaG7yDSYcTZ5mTXd3ax+YnNtDe0d5VLjFD8uWKWPLKE5NzoW+DS6XZzvrGNwvQkn3IdnjZ89SWIvwP8HfibiNzpvcHTKv0RkAj8Rx+e8zPABOBaY8xxz3MdBI4BnwWev9KBIuIAfg6sN8b8g9emjX14fWUDYwxv/vFN1n11HTXv1Phsu6rkKkqeLyFvSp5NtbNXh9vNjvIaLrW00+pyMzErOi5IRrugg9gYUyoijwCPAIeADgARuQRkYXUrfM0Ys70Pr38LsLMzhD2vc1JEtgG30kMQA0uAKcDn+vB6ymYV+ysovauU05tP+5TnTspl1fOruPpdV9tUM/t1uNxsK6+murUDgIMX68lNiiczcXhO16m69WkApjHmMRHZAvwbMBfIwepO+Buw2hizoY+vPxV4OUD5YeCDvRy70PM1UUR2ArOAGuA3WP8htFzxSDXoGioa2PCNDRz46QHo7gYmKTuJJY8uYdZnZxETxcv4tHtCuMYTwgDT8tI0hKNEn0fCG2M2ErqP/9lY4emvGquV3ZPOOQ3/B/gecD9QjHXhrgj4hyscpwZRR0sHO1fvZOtTW2lv7O4HdsQ6mP2vs1n80GKSspN6eIbhr83pZmt5FXVtzs4R8zoAACAASURBVK6y60eka7dEFOnLnXWfAA4YYw72sM91wA29jDX2ZwKUBXNVonPZgV8ZYzpvLHlVRGKAp0VkijHmSIA63gncCTBmzJg+VFP1hTGGw789zLqvraPudJ3Ptmveew0rn1tJ7rW5NtUucrQ6XWwtq6a+vTuEZ4xMZ0KmhnA06csaKj8D3t/LPrcAP+3Dc9ZgtYr9ZRG4peytc9aXtX7lazxfZwQ6yBjzkjGm2BhTnJcXnReEwu3s7rP8dNFP+cOH/+ATwnlT8/jYmo/xkb98REMYaHG62OIXwjPzMzSEo1Cob9KPIXAL90oOY/UT+5sCXNaaDXAsAV6vszXtRg2q+rP1bPj6Bl7/xes+5cm5ySx9fCkzPz0TR6yunwbQ3OFia1kVjR2urrLi/AzGZETfcD0V+iC+ht5bst7+DDwnIhOMMe8AiMg4YAFWn29P/o41/vhmfMcul3i+7ulDPdQAdDR3sP257Wx7Zhsdzd0XmxxxDm78txu56Rs3kZiZaGMNI09ju5NmpxXCAswelXnZuGEVPXoMYhH5L7+i93uC0l8MMAZr9rW/9uH1fwR8EXhZRL6B1bp9HCgDXvSqx1jgBPCYMeYxAGNMlYg8BXxTROqxbuwoxpqI6OfeQ+JUeBi34Y3/9wbr719PfXm9z7ZJ75/EymdXkj0xUM+TGpGSwJzRWew+V8vsUZmMTtP/qKJZby3iO7y+N1j9rgH7Xj3bdwF3B/vixpgmEVmGdYvzL7EaB+uxbnFu9NpVsMLe/3PtY1i3Xn8BuA+oAJ7FCnMVRuU7y3nlrlc4u+usT/nI6SMpWV3C+KXjbarZ0DE6NZGbJ+TpOnOq1yDu/GsS4B3gBaw77Py5gJrebkkOxDOnxG297HOKACMpjDEG66aPnm78UCFUV1bH+vvX88Z/v+FTnjIihWVPLmPGJ2fgiNF+YH/1bR3EOhwk+42V1hBW0EsQG2O6bn8SkUeBjd5lKnq0N7az7Vvb2P7cdpwt3Vf5Y+JjmHv3XBZ9fREJ6Qk21jBy1bV2sLW8mjiHsGhMDkkavspPX25xfjScFVGRybgNr//ydTZ8fQMN53zneprygSmseGYFWRN6u/cmetW0drCtrIp2t6HNBTvP1rBkTI7OoKZ89GvUhOemiVwgYBMo2CksVWQ7s/UMpXeXcm7POZ/yUTNHUbK6hLE3jbWpZkNDdUs728qr6fBM7RnnEJ1LWAXU18VDrwOexpr790qfQ7sWFFVDU83JGtZ9bR1Hfuc7lDs1P5XlTy1n+iemIzolY4+qmtvZdrYap1cILyzKJisx3uaaqUjUl1ucJwGdM6utBd4HvA5cAGZitZA3Yk3MroagtoY2tj61lR3P78DV1n2jQWxiLPPuncfC+xcSn6pB0pvK5ja2l9fgMlYIx8dYk7rrBD7qSvrScv0mEAfMNsa8ISJu4E+eGdlSsOYhfje+Q97UEOB2uTnwswNseHADTRd8B75M+/A0lj+9nMyxmTbVbmi52NTGjrPVdC44nRDjYGFRNhkJGsLqyvoSxEuA/zPGeI9bEugaD/xZ4CDWGN47QlVBFV6nXj1F6d2lnD9w3qe8YE4BJatLKJpfZFPNhp7zja3sPFdD52pPiTEOFhXlkJagPXWqZ335DcnFWjmjkxPoujHeGOMUkY3o9JNDQvWJatZ+ZS1v/ektn/K0gjRWPL2C6/75Ou0H7qOTdc1dIZwUa4VwaryGsOpdX35LqvFdC+4S1m3N3trRVZwjWmtdK1ue3MKu7+zC1e7VD5wUy4KvLWD+ffOJT9F+4P6YMyqLHWeraexwsagwmxQNYRWkvvymnADGef28F1gpIiOMMRc9/cS3AicDHazs5Xa52ffjfWz85kaaK5t9tl3/setZ/tRy0gvTbard8BDjEOYWZNPhcpMUxauNqL7rSxCvAb4qIimeW5l/CLwH2C8i27GWKhoL3Bv6aqqBeGf9O5TeXcrFNy76lBfOK+TmF26mYE6BTTUb2upaO8jwGwkR6xBiHRrCqm/6EsQ/Ao4CSUCTMeavInIX1mKitwHNwDP0bRVnFUZVb1ex5r41vP2Xt33KM8ZksOKZFUz90FS9uaCfTtY2s/9CHVNz07g2J7X3A5TqQV9uca7AWh/Ou+w/ROT7WBfyLnom4VE2a6lpYfPjm3ntu6/hdnbPjx+XEsfCBxYy7555xCXpcKr+OlHTxOsXrWk/D19qIDHWwVid0F0NwICvJhhjXFg3dSibuZ1u9ry4h1cffpWWKt9FrGfcMYNlTy4jbXSaTbUbHo5VN/JGZfecG5kJcYxK1bmE1cCE9LKuiKwCHjfG3BjK51W9O156nDX3rKHySKVP+ZhFYyhZXcLoWaOvcKQK1tGqRg5f6g7h7MQ45hdmE6/TfqoB6sstztmA0xhTH2DbPODfgZtCWDcVhMo3K1lz7xqO/913QZLMcZmsfHYlk2+brP3AA2SM4a2qRt6s6l6rICfJCuE4h4awGrheg1hEbgO+hWfomoi8AXzWGLNLREYAP8C6iUOAA1hLFakwa65qZtOjm9j9g90YV3fXfHxaPIseXMTcL88lNlHHsQ6UMYYjlxo5Wt0dwnnJ8cwryCJWQ1iFSG9r1i0Cfovv6hjXA38XkSXAX4AirBWVHzbG/DFM9VQerg4Xu3+wm02PbqK1prV7g8DMT89k6eNLSR2pV/FDwRjDocoGjtV0z78xIjmeuQXZxOpdhyqEemsy3YUVwg8AP/GUfQ5rrbgNWHfafRH4oTFGl68PI2MMx/52jDX3rqHqaJXPtnFLxlGyuoT8Gfk21W54auxwcaK2O4TzUxK4cXQWMRrCKsR6C+K5wHpjzDNeZU+IyFKsSYDuNMb8JOCRKmQuHrpI6T2lvLP2HZ/yrKuyWPXcKq699VrtBw6DtPhY5hVks+NsNSM9IezQf2cVBr0FcR7Wrcz+9mAF8R9CXSHVramyiVcffpW9L+7FuLv7gRPSE7jpoZuY88U5xOrMXmE1MiWBm4pyyEyM0xBWYdPbX3Es1h1z/poBjDG1Ia+RwtXuYtd3d7H58c201bV1lYtDmPXZWSx5dAkpeSk21nB4chtDu8tNot/intlJOgmSCi9tTkUQYwxH/3yUtfetpfp4tc+2CSsmULK6hBHTRthUu+HNbQx7Kmqpbu3gpqKcy5a9VyqcggniOzwjJLyNAxCRDQH2N8aY5QOsV9Q5//p51tyzhpMbfCevy7kmh1XfXsXV77la+4HDxG0Mr52r4Vyj9eljS1kVS8fm6o0aatAEE8Tj8J3+0tuSAGU630QflW0v42eLf+YzL0RiZiKLH1nM7M/PJiZeW2fh4nIbdp2r4XxTdxdQfkoCcToyQg2i3oJ46aDUIsod+p9DXSEsMcLsL8xm8cOLSc7RiWTCyeU27DhbzcXm9q6yq7NSmJaXpp8+1KDqMYiNMZsGqyLRrGJPRdf3t/33bUz9p6k21iY6ON1udpytodIrhK/NTmFKroawGnx6sc5mbqebiv3dQTx28VgbaxMdOtxutpfXUNXSHcKTc1KZlJOqIaxsoUFss8ojlThbnACkF6Xr7clh1uFys628murWjq4yndxd2U2D2Gbn9pzr+n50sU5VGW5Hqxt9Qvi6vDSuztYQVvbS8Tk20yAeXFNy08hPSQBg+oh0DWEVEbRFbDMN4sHlEOHG0VlcbG7TlTVUxNAWsY1c7S4uvN69ytSoWaNsrM3w1OFy47+UYoxDNIRVRNEgttHFQxdxtbsAyByfqeOGQ6y5w8WG05d4y2tlDaUiUTArdGQCXwLmYN01txP4vjGmLsx1G/a0WyJ8mjqcbCmrprnDxZtVjThEdGSEili9rdCRCbwGXEX3Kh3vAW4XkRt19rWB0SAOj8Z2J1vKqmjpvFsRa25hpSJVb10TXwMmAm96vr8fOOop+1p4qzb8aRCHXkObk81eIewQmFeQxeg07RNWkau3IH4vcBaYY4x51hjzLawuigrgfaGogIgUicjvRaROROpF5I8iMqYfz/OAiBgR2RqKeoWbs9XJxTcudv08aqZeqBuourYONpdV0eoJ4RiBeQXZ5OuFORXhegvi8cBfjDFdk8MbYxqBP3PlGdmCJiLJWGvfTQJuBz4OXA1sFJGgZz4XkQnAg8DF3vaNFBcOXuia6Cf76mwSMzUsBqK2tYMtZdW0uTpDWJhfmM1Iz5hhpSJZbx1nycD5AOUXgKQQvP5ngAnAtcaY4wAichA4BnwWeD7I5/lP4NfAtQyRsdHaLRE6Na0dbC2rosOznFSsQ1hQkE1Osq6soYYGu4ev3QLs7AxhAGPMSWAbcGswTyAi/wzMxFppesjQIA6NzqksO0M4ziEsLNQQVkNLMK3HGSLyCf8yABH5ON2jKboYY34R5OtPBV4OUH4Y+GBvB4tIFrAa+KoxpnoozZzlE8SzNYj7K8YhzB6VyfbyamJEWFCUQ1ZinN3VUqpPggniWwncOhXgZ1c4JtggzgZqApRXA1lBHP8s8HYP9YhI7U3tVB6utH4QGHWDXqgbiLzkBOYVZBMf4yBTQ1gNQb0F8S8I/9JHgZ6/16atiCwCPgHMNP73sPZ83J3AnQBjxvR5cEZInD9wHuP5KJ03OY/4VP0Y3RduYy5b2n6EXpRTQ1hvK3TcEebXr8FqFfvLInBL2duLwE+Acs+NJ2CdT4zn5xZjTJv/QcaYl4CXAIqLi21ZX0/7h/vvfGMrr1+sZ2FhNil6k4YaJnq8WCcinxCR68P4+oex+on9TQGO9HLsZOBzWIHd+VgAzPV8//nQVTO0vJdGGlWs3RLBOtfQyo6zNTR1uDy3LzvtrpJSIdFbk+JnwCPAwTC9/p+B50RkgjHmHQARGYcVqPf3cmyghU1fAGKw5sY4HmB7RNAWcd+VN7Sw+1xtdz+W6HLhaviw+7Pdj4AvAi+LyDew/rYeB8qwuh4AEJGxwAngMWPMYwDGmFf9n0xEaoHYQNsiRVt9G5eOXgKsFZvzp+fbXKPId6a+hT0V3dOapMbFsLAoh+S4GBtrpVTo2DqO2BjTBCzDGvnwS6ybMk4Cyzx38HUSrJau3eOeB6xif0VXU27E1BHEJetV/p6cqmv2CeG0+BgWjdEQVsOL3S1ijDFngNt62ecUQYykMMYsCU2twse7W0L7h3t2sraZ/Re6Z1tNj49lYVE2ibEawmp4CSaIM/s6CY8nXFUA3hfqtH/4yk7UNPH6xfqunzMSYllYmENC7JD/UKTUZYIJ4i97HsEyQT5vVNILdb2r8AxR65SVGMeCQuuGDaWGo2ACsx7QCeBDoKWmherj1QA44hyMvH6kzTWKTCNTEhidmsi5xlayPSEcpyGshrFggnh150gFNTAV+7q7JUZeN5LYBP3gEIhDhDmjM3mrqpGrs1OIc2gIq+FNk2AQ6YW6wDrvUPeetMkhwpTcNLuqpNSg0qbGINILdZczxnCosoHDlxouW/ZeqWihLeJBdHb32a7vC2YX2FiTyGCM4eDFek7UWgvAaCtYRSsN4kHSVNlE3WlrTGxMQgx5U/NsrpG9jDHsv1DPqbquVbiob+vAGMNQmldaqVDobfY17boIkYq93d0S+TPyiYniO8OMMew9X8eZ+paussK0RIpHZWoIq6ikLeJBouOHLW5j2FtRS1lDa1fZmPQkZuZnXDbHsFLRQoN4kGgQWyG8u6KWs14hPDYjiZkjM7QlrKKaBvEgifYgdrkNr1XUUNHYPVf/+IxkZoxM1xBWUU+DeBA0VDTQcLYBgLjkOHIn5dpco8G373ytTwhflZnM9SM0hJUCHUc8KHwu1N2QjyMKJ66ZkJVCrCd0r85K0RBWyou2iAdBtHdLAOQkxTO/MIvK5nYm5aRqCCvlRYN4EGgQW3KTE8hN1tWWlfIXfZ+RB5kxJuqCuN3lZsfZahradXFPpYKhQRxmDWcbaLrQBEB8ajw51+TYXKPwane52VpWTUVjG1vKqmjUMFaqVxrEYeYz49qsUYhj+PaNtjldbCmroratA4BWp5uqlnaba6VU5NM+4jCLlm6JVqeLLWW+3RE3jMxgbEayjbVSamjQIA6zaAjilg4XW8qraGx3dZXNytcQVipYGsRhZIzh3G6vIJ49/IK4ucPqjmjqsEJYgOJRmRSlJ9lbMaWGEA3iMKo9VUtLtTXDWGJmIlkTsmyuUWg1tTvZUl5Ns1cIzxmdSUGahrBSfaFBHEb+3RLD6SaGFqeLzWVVtDjdADgE5ozOYnRqos01U2ro0VETYTSc16hLjHGQmxQPWCE8t0BDWKn+0hZxGA3nNepEhFmjMhGpY0x6EiNS9I45pfpLgzhMjNtwbu/wHjHhEKF4VKbd1VBqyNOuiTCpPlFNW5017WNybjIZYzJsrtHA1LS2c6iyXldaVioMtEUcJsPpQl1VSzvbyqtxug3GwLS8tCF9PkpFGm0Rh8lwuVB3qbmNbWVWCAOcrmum1TNSQikVGtoiDpPhcKHuYlMbO87W4PJ0RyTEOFhYmE1SFK9ArVQ4aBCHgdvlpmLf0A7iC01t7DhbjachTEKMg0VF2aQnxNlbMaWGIQ3iMKh6u4r2RmvWsdT8VNJGp9lco76paGxl17marhBOjHWwqCiHtHj9dVEqHPQvKwyG8oW6sw2tvHauhs6xEUmxMSwqyiZVQ1ipsNG/rjAYqhfqzja08Nq52q4QTomzQjg5Tn9NlAonHTURBj4zrg2h/uHkuFhiPRPXp8bFsKgoR0NYqUFgexCLSJGI/F5E6kSkXkT+KCJjgjiuWEReEpG3RKRZRM6IyK9FZPxg1PtK3E435/ef7/p5KAVxVmIcCwqzyU6MY9GYHJJ1dIRSg8LWIBaRZGADMAm4Hfg4cDWwUURSejn8w8BU4D+AdwH3AzOBPSJSFLZK96LySCXOVmuVivSidFJHptpVlX7JTopn8ZgckmI1hJUaLHZ/7vwMMAG41hhzHEBEDgLHgM8Cz/dw7DPGmErvAhHZBpz0PO9DYalxL4bSihwna5vJToojw29I2lC6uKjUcGB318QtwM7OEAYwxpwEtgG39nSgfwh7yk4DlUBBiOsZtKESxG9XN7L/Qh1by6ppaNOVlpWyk91BPBU4FKD8MDClr08mIpOBEcCbA6xXvw2FIH6rqpFDlQ0AtLncHL7UYHONlIpudndNZAM1AcqrgT6tKyQiscAPsVrEPxl41frO1e7iwusXun4eNSuyhq4ZY3izqpG3qhq7ynKT4ikeNbRnhlNqqLM7iAECzavYn07K7wHzgfcYYwKFu/XEIncCdwKMGdPr4Iw+uXjoIi7PSsaZ4zNJzomcVYyNMRy+1MDb1U1dZXnJ8cwryCLWYfcHI6Wim91/gTVYrWJ/WQRuKQckIk9hheu/GGPW9LSvMeYlY0yxMaY4Ly+vT5XtTaR2SxhjeKPSN4RHpiQwvyBbQ1ipCGB3i/gwVj+xvynAkWCeQEQexBq69m/GmF+GsG59FolBbIzh9Yv1vFPb3FWWn5LAjaOziHHo6AilIoHdzaE/A3NFZEJngYiMAxZ4tvVIRP4NeAJ40Bjz3TDVMWiRFsTGGPZfqPMJ4dGpicwt0BBWKpLYHcQ/Ak4BL4vIrSJyC/AyUAa82LmTiIwVEaeIPORV9mHgBeAVYIOIzPV69HnExUA5W51cfONi18+jZkbGhTrvDvjCtETmjM7EoeOElYootnZNGGOaRGQZsBr4JdZFuvXAXcaYRq9dBYjB9z+Omz3lN3se3jYBS8JU7YAuHLyA27NyRfbV2SRm2r+0vIgwc2QGncvMzcrP0Js1lIpAdvcRY4w5A9zWyz6n8BtJYYy5A7gjXPXqq0jrlugkIszMz0DQO+aUilR2d00MG5EQxC634Z2apstWWnaIaAgrFcFsbxEPF3ZPfelyG3aeq+FCUxv17U6mj0jX8FVqiNAWcQi0N7VTecQz9YUM/oU6p9uw42w1F5raAHintpnznu+VUpFPW8QhcP7AeYxngbe8yXnEp8YP2ms73W62l9dwqaW9q2xSTir5KQmDVgel1MBoEIeAXf3DHS43289WU9XS0VU2JTeVSTlDa7FSpaKdBnEIVOyp6Pp+sNaoa3e52VZeTU1rdwhPy0vjmuyhNRG9UkqDOCQGu0Xc5nKzrayKWq95hK8fkc7ErN4WNVFKRSIN4gFqq2/j0tFLAEiMkD89P6yv1+p0sa28mjqvEJ4xMp0JmRrCSg1VGsQDVLG/ous+4hFTRxCXHNfzAQPU7nLT4nR1/TwzP4NxGZEz3aZSqu90+NoAeXdLDEb/cHpCHAsLc4iPEYo1hJUaFrRFPEDeF+oGa8REZmIcJeNHEBej/48qNRzoX/IAhftCXVO7k1qvkRGdNISVGj70r3kAWmpaqD5eDYAjzsHI60eG9Pkb2p1sKqtia1kVdQHCWCk1PGgQD0DFvu5uiZHXjSQ2IXQ9PfVtHWw+U0Wr002727D9bA0ud6Dl/ZRSQ532EQ9AuC7U1bV1sLWsmjaXNb9xjFhzCeuqGkoNTxrEAxCOGddqWzvYWl5Fu8tq/caKML8wi9xknTtCqeFKg3gAQn2hrrqlnW3l1XR4uiBiHcKCwmxykgZvEiGl1ODTIO6npsom6k7XARCTEMOIaSMG9HxVnhB2ekI4zhPC2RrCSg17GsT9VLG3+0Jd/ox8YuJi+v1cl5rb2FZeg8uzskZ8jLCwMIfMxPDepaeUigwaxP0Uym6Jisa2rhBOiHGwsCibjAQNYaWihQZxP4UyiKflpdHhNpxvbGVhUTbpGsJKRRUN4n4KZRCLCDeMTKc1N5Wk2P53cSilhia9oaMfGioaaDjbAEBcchy5k3L7dHx1S/tlKy2LiIawUlFKg7gffC7U3ZCPIzb4f8ay+hY2nali34W6y8JYKRWdNIj7ob/dEmfqmtldUYsBTte18FZVYxhqp5QaarSPuB/6E8Sn6prZd76u6+e0+FjGZ+pcwkopDeI+M8b0OYjfqWniwMX6rp8zEmJZWJhNgvYJK6XQIO6zhrMNNF1oAiA+NZ6ca3J63P94TRMHvUI4MyGWBUU5JOh8wkopDw3iPvKZcW3WKKSHGdHermrk0KWGrp+zEuNYUJhNvIawUsqLBnEfBdst8ealBt70uhiXkxTH/IJsXVlDKXUZDeI+Cmbqyzanm3dqm7t+zk2KZ35hFrEODWGl1OU0Gfog2At1CbHWfBHxMcKI5HjmF2ZrCCulrkhbxH1Qe6qWluoWABIzE8m6KuuK+2YkxLF4TC7JsTG6soZSqkfaTOsD/9awiBWwxhga252X7Z8WH6shrJTqlQZxHwRao84Yw74LdWw8fYma1na7qqaUGsI0iPugYk/3HBOji0djjGHv+TpO17XQ4TZsLasO2DJWSqmeaB9xkIzbcG5vd4s4f9YodlfUUt7Q2lU2KjWRlAGs1KGUik62t4hFpEhEfi8idSJSLyJ/FJExQR6bKCLPikiFiLSIyA4RuSkc9aw+UU1bXRsAyfmpvB1nfEJ4XEYSs/IzuvqNlVIqWLYGsYgkAxuAScDtwMeBq4GNIpISxFP8BPgM8BDwXqACKBWRGaGua2f/sMQ5GPe9lZxrbOvaNiEzmRtGaggrpfrH7q6JzwATgGuNMccBROQgcAz4LPD8lQ4UkenAPwP/Yoz5qadsE3AYeAy4JZQVPbfnHJIQw9jvriRmWl5X+cSsFK7LS9MQVkr1m91dE7cAOztDGMAYcxLYBtwaxLEdwP94HesEfgOUiEhCKCtacfAi4/6zhLSbuntNrsnWEFZKDZzdQTwVOBSg/DAwJYhjTxpjmv3KDwPxwMSBV8/idrlpHp1M6vzCrrJJOalMzdUQVkoNnN1BnA3UBCivBq5821rvx3Zuv4yI3Ckie0RkT2VlZVCVrHq7ispfHOLSL94AYEpOKlM0hJVSIWJ3EAMEWrgtmIST/hxrjHnJGFNsjCnOy8vradcubXVtjJo1igvP7qL9xdeZlJsW1HFKKRUMuy/W1RC45ZpF4Naut2og0DC3LK/tIVE4t5A799yJs81Ja01r7wcopVQf2N0iPozV1+tvCnAkiGPHe4bA+R/bDhy//JCBiU2IJTU/NdRPq5SKcnYH8Z+BuSIyobNARMYBCzzbejs2Dvig17GxwIeANcaYtisdqJRSkcTuIP4RcAp4WURuFZFbgJeBMuDFzp1EZKyIOEXkoc4yY8wBrKFrL4jIp0VkOdbQtfHAw4N4DkopNSC2BrExpglYBrwN/BL4NXASWGaMafTaVYAYLq/vJ4GfAk8AfwWKgJuNMfvCXHWllAoZuy/WYYw5A9zWyz6nCDAawhjTAtzjeSil1JBkd9eEUkpFPQ1ipZSymQaxUkrZTINYKaVspkGslFI2E2MCTdcQHUSkEjjdh0NygUthqs5QEM3nH83nDtF9/v0597HGmOAmsyHKg7ivRGSPMabY7nrYJZrPP5rPHaL7/Afj3LVrQimlbKZBrJRSNtMg7puX7K6AzaL5/KP53CG6zz/s5659xEopZTNtESullM2iLohFpEhEfi8idSJSLyJ/FJFAK30EOjZRRJ4VkQoRaRGRHSJyU4D9HCLygIicEpFWEXldRHqc2Ggw9PfcRaRYRF4SkbdEpFlEzojIr0VkfIB9T4mICfB4f3jOKjgDfN8DnY8RkRl++0Xk+w4Deu8f6eH8W/32jdT3vlBEvuv5e2321GlckMcG/Z6KyGc8fyNtInJURD4XdCWNMVHzAJKBY1grR78fuBV4AzgBpARx/K+BWuAzwHLgj0ALMMNvvyeBNuA+YCnW3Mpu4N1D8dyB54BtwBeAxcA/A28CVUCR376ngFeAuX6PrKF47p7jDdZ0q/7nlBzp73sI3vvCAOe9HOgAfhvp772nXkuAC8DfgFLP+zkuyGODek89meD27L8Ua2peN/D5oF7Hzn8gG96QLwMuYKJX2XjACdzTy7HTPW/gJ73KYoGjwJ+9ykZ43rhH/Y5fDxwcouee3GpM7AAACK1JREFUF6BsrOcX7TG/8lPAr+x+r0N17p59DfBEL/tE5PseivMP8Hwf9/ybvCfS33tPvRxe33862CAO9j315MBF4Od++/0X1o0gcb29VrR1TdwC7DTGdK1nZ4w5idXauzWIYzuwVgXpPNaJtSpIiYgkeIpLgHjgV37H/wq4LtDH+UHS73M3xlQGKDsNVAIFIa5nOAzkfQ9WpL7vEPrzvx2rhVkamuqFlzHG3c9Dg31P5wF5Afb7JZADLOzthaItiKdifTzzdxhr0dHejj1pjGkOcGw8MNFrvzYuX7z0sOdrb68TLgM598uIyGSsFsObATa/z9MX1yYiO+3uIyQ05/55z/k0i8gGEVkU4DUi8X2HEL73IlKI9dH7156GiL9Ie+8HItj3tHMBZP9/46Df+2gL4mygJkB5NZA1gGM7t3d+rTWezyY97DfYBnLuPjyLtP4Qq0X8E7/NfwG+hNWa+CjQCvxJRD7W1wqH0EDP/VdY/eMrgDuxWjkbRGSJ32tE4vve+dohee+xuiUcwM8DbIvE934ggn1PO7/6/xsH/d7bvlSSDQINnL5sGaYr7BPMscHuZ4dQ1et7wHysPkKfXz5jzJd8nlzkT8BO4Cku/+g2mPp97saYj3v9uEVEXsZq/TxB98fOSH7fIXR1+wSw3xhz8LIXiNz3vr/68jfPFfYNSrS1iGsI/L9TFoFbDN6qezi2c3vn1ywR8X+z/PcbbAM59y4i8hRWq/BfjDFretvfGOMCfgcUisioYF8nxEJy7p2MMQ1Yi9XO9iqO1PcdQvfezwEmEbg1fJkIee8HItj39Eot32y/7VcUbUF8mO7+HG9TgCNBHDteRJIDHNtOdz/SYSABuCrAfgTxOuEykHMHQEQeBO4HvmyM+WUfXnvALYYBGvC5B+DfWorU9x1Cd/63Y420+O8+HGP3ez8Qwb6nnX3B/v/Gwb/3dg8tGeRhLHdh/SJN8CobhzUa4t5ejp2B9ct0u9+wlf/f3r3F2DXFcRz//tCWuFRCPLg/qCDiwaXa0ExDEKGEiIwQ98RdE9EXwbRu8SAuiVtdGyEuaRNtEyoumdYL4lIa6hKCIQQNdQnVxt/Dfw17Ts/UOTPDPmf8PsnJmdl7nbX3OuvkP2v2Xmf9VwNLm0x56Wt4/QvAqm5seyl7eWn/VW0edwvgdeCzbm17k/q2Az4Hlnd6v49V+8kb0muAxd3U903OaSTT1zbZp8AE8n7Jww3lHijv2cR/PFbdb8x/3AlbkyPXVeS0nROAt4FPgG0q5fYoH9xrG17/BPmv3PnkpPaF5A2JAxvK3Vy2X0FOJr+HnHM7qxvbDvSW83+WjSfr71cpd1p5j84k76z3Ai+XD35vl7b9SuB+8kssM8lR4Sryv6AZnd7vY/G5L/tOLv148jDH6Mi+r5zfKeVxTzmni8rvPZUyG4AHR9KnwIVl+w2l3HXl90taOr+636AaOmR3YBHwI/AT8DQNfx3J0UIAcxu2bwXcCnxdOudVYGaTY2wOXE1m/1gHvAOc0q1tBxaUbc0e/ZVy04CXyDmm64G15OjhmC5u+yxyvu13pU1rgCXA1G7p99F+7su+xWxidNfJfV/Or5XPbwALRtqnwAXAh6XcR8DFrZ6fV18zM6vZ/+1mnZlZx3EgNjOrmQOxmVnNHIjNzGrmQGxmVjMHYjOzmjkQm5nVzIHYOtIm8qQNPs6ulG2WV+1XSR9KuqusodvKMfdsUs8GSd9IWiZprBaRH2xf/1jVZ93t/7gMpnWXecNsX9lk23Kgv/y8I3A0uY7wqZKmRcTHLR5zLXB7+XkSuZjL8WQmljkRcUuL9Zi1xIHYOlpEzG2jeH+1vKQJ5PoYR5JfUz2nxXp+aDyupF7gcWCepLtj40wtZiPmSxM2bkXEeuC+8uvUUVb3JPAzmRF5SOobSZMlzSkplL6Q9LukbyUtkTStoezZkgbXFehpuAwyt6HsoZIWSvq61Dkgab6knUfZFuswHhHbeDeW6+EO1rW+Yfu+ZBr1FeSC8d+Ti+ycABwraVZELCtlV5KXW/rIhWQWVOrp/+tA0jnkqm/ryEWGBoAp5Mp/s8qlls/HoE3WARyIraM1jhKLTyNiQQuv3YLMJgK5Ut5onE4uJ/kt8EHDvtXAzhHxXcPxdwVeA24DlgFExEpgpaQ+sh1zm5z33sB8Mj19T0R8Wdl3BPA8cAdw0ijbZB3Cgdg6XV+TbcsZOpIcNLMSuHcgk1hOIZewvLGNY25fqWcSsD9wHLkG8QUR8Vu1cESsbVZJRHwhaSFwmaTd2xjBXkQuNj67GoRLnS9JWkKOireNTNtkXc6B2DpaRLST4LKnPCCD5gCZbfqmiBhoo57JbPwHYB1wYkQ81+wFkg4DZgPTycwOExuK7EJm9WjF9PLcI+mQJvt3ItfJ3Rt4o8U6rYM5ENt4Mq/NWRbD+Swi9gSQtB1wFJn25ilJ0yNiSA4ySSfxd7aW54GPgV/IDA0zyT8Ok9o4/g7lec4/lNumjTqtgzkQm21CRPwILJL0K3kj7hFJh8TQjArXkyPwgyNidfX1kubz9yi9VYOXOiaX49s45+lrZi2IiGfIG24HkfnrqvYC3msShDcDDh+myj/IywvNvFKeZ4zsbK3bOBCbte6a8jyvzMgY9CkwpTq/V5LI68xD5hxXrAF2G2bfneQUudvKDIohJE2U5CA9jvjShFmLIuJ1SYvJTMjnkVPMIKen3Qu8JWkRGUQPI4PwUjIBaaMXgV5JS8kbbhuAFRGxIiLel3Qu8BDwrqRlZFLKCeT85BnkNLp9/p2W2n/NI2Kz9vSRXw65RtKWABExn/z69FfAWeSc4wHgUODNYeqZTX5leio50r4eOGJwZ0Q8Sl4GeQw4ALgUOIO8DLKQXEPDxglncTYzq5lHxGZmNXMgNjOrmQOxmVnNHIjNzGrmQGxmVjMHYjOzmjkQm5nVzIHYzKxmDsRmZjVzIDYzq9mfU4Xrf6qjjGwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.size'] = 16\n",
    "plt.subplots(1, 1, figsize=(5,5))\n",
    "\n",
    "rand_probs = [0 for _ in range(len(test_labels))]\n",
    "rand_fpr, rand_tpr, thresholds = metrics.roc_curve(test_labels, rand_probs)\n",
    "\n",
    "fpr, tpr, thresholds = metrics.roc_curve(test_labels, b2_pred,\n",
    "                                         pos_label=1)\n",
    "plt.plot(fpr, tpr, linewidth=3, color='purple')\n",
    "plt.plot(rand_fpr, rand_tpr, linestyle='--', linewidth=3, color='lightblue')\n",
    "plt.title(\"ROC Curve\", fontsize=20)\n",
    "plt.xlabel(\"FP Rate\", fontsize=20)\n",
    "plt.ylabel(\"TP Rate\", fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "18Sjo2KpZz-g"
   },
   "source": [
    "### B3: Negative and positive sentiment lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4dE-TRbUZz-g"
   },
   "outputs": [],
   "source": [
    "b3_pred = test_features.apply(lambda row: 1 if row.negative_conc==1 or row.positive_conc==1 else 0, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fk3omCnLZz-h",
    "outputId": "80a37719-d626-4b05-c227-e6346f6f47bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of b3, test:\n",
      "F1: 0.32\n",
      "Precision: 0.22\n",
      "Recall: 0.57\n",
      "AUC: 0.68\n",
      "Accuracy: 0.76\n",
      "Confusion matrix:\n",
      " [[2209  129]\n",
      " [ 603  173]]\n"
     ]
    }
   ],
   "source": [
    "print('Performance of b3, test:')\n",
    "print('F1:', round(metrics.f1_score(test_labels,b3_pred),2))\n",
    "print('Precision:', round(metrics.precision_score(test_labels,b3_pred),2))\n",
    "print('Recall:', round(metrics.recall_score(test_labels,b3_pred),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(test_labels,b3_pred),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(test_labels,b3_pred),2))\n",
    "print('Confusion matrix:\\n', \n",
    "      metrics.confusion_matrix(test_labels,b3_pred).transpose())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dWlwf0t7Zz-i"
   },
   "source": [
    "### B4: Semi-automated bias lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6_jPE0fNZz-i"
   },
   "outputs": [],
   "source": [
    "b4_pred = test_features.bias_lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZNVI2wgeZz-j",
    "outputId": "c6d39819-69ab-4be4-a462-6bf1d399ea04"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of b4, test:\n",
      "F1: 0.2\n",
      "Precision: 0.62\n",
      "Recall: 0.12\n",
      "AUC: 0.56\n",
      "Accuracy: 0.91\n",
      "Confusion matrix:\n",
      " [[2790  266]\n",
      " [  22   36]]\n"
     ]
    }
   ],
   "source": [
    "print('Performance of b4, test:')\n",
    "print('F1:', round(metrics.f1_score(test_labels,b4_pred),2))\n",
    "print('Precision:', round(metrics.precision_score(test_labels,b4_pred),2))\n",
    "print('Recall:', round(metrics.recall_score(test_labels,b4_pred),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(test_labels,b4_pred),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(test_labels,b4_pred),2))\n",
    "print('Confusion matrix:\\n', \n",
    "      metrics.confusion_matrix(test_labels,b4_pred).transpose())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8PyQBOEWZz-j"
   },
   "source": [
    "## 3 XGBoost optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 790,
     "status": "ok",
     "timestamp": 1621016167238,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "iczxxoLAZz-k",
    "outputId": "1f40def4-e416-4f1d-e9f9-0d9c0d1ca580"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Biased words: 5.0 %\n",
      "Neutral words: 95.0 %\n"
     ]
    }
   ],
   "source": [
    "print('Biased words:',round(len(y4[y4['label4']==1])/len(y4)*100,0),'%')\n",
    "print('Neutral words:',round(len(y4[y4['label4']==0])/len(y4)*100,0),'%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "executionInfo": {
     "elapsed": 781,
     "status": "ok",
     "timestamp": 1621016167239,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "2QT63zODLnEM"
   },
   "outputs": [],
   "source": [
    "def f1_eval(predt: np.ndarray, dtrain: xgboost.DMatrix):\n",
    "    y = dtrain.get_label()\n",
    "    predt_binary = np.where(predt > 0.5, 1, 0)\n",
    "    return \"F1_score\", metrics.f1_score(y_true=y, y_pred=predt_binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Aq4j6un0Zz-k"
   },
   "source": [
    "### 3.1 Hyper-parameters tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1yquPMPzZz-l"
   },
   "source": [
    "We don't train full gridsearch, instead, we train different mutually dependent hyper-parameters together:\n",
    "- max_depth & min_child_weight\n",
    "- subsample & colsample_bytree\n",
    "- eta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 567,
     "status": "ok",
     "timestamp": 1621012137332,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "kHJXSGFBZz-l",
    "outputId": "02fbc71c-6943-4f30-9eaa-68bd7b069d8b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37.252043596730246"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scale_pos_weight = len(train_labels[train_labels['label4']==0])/len(train_labels[train_labels['label4']==1])\n",
    "scale_pos_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "executionInfo": {
     "elapsed": 600,
     "status": "ok",
     "timestamp": 1621012139062,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "qOipcdtyZz-m"
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'max_depth':6,\n",
    "    'min_child_weight': 1,\n",
    "    'eta':.3,\n",
    "    'subsample': 1,\n",
    "    'colsample_bytree': 1,\n",
    "    'objective':\"binary:logistic\",\n",
    "    'disable_default_eval_metric': 1,\n",
    "    'seed': 42,\n",
    "    'tree_method': 'hist',\n",
    "    'scale_pos_weight': scale_pos_weight\n",
    "}\n",
    "\n",
    "num_boost_round = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "executionInfo": {
     "elapsed": 474,
     "status": "ok",
     "timestamp": 1621012141389,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "rHFtj3JhZz-m"
   },
   "outputs": [],
   "source": [
    "def f1_eval(predt: np.ndarray, dtrain: xgboost.DMatrix):\n",
    "    y = dtrain.get_label()\n",
    "    predt_binary = np.where(predt > 0.5, 1, 0)\n",
    "    return \"F1_score\", metrics.f1_score(y_true=y, y_pred=predt_binary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "executionInfo": {
     "elapsed": 785,
     "status": "ok",
     "timestamp": 1621012143202,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "PabzCbCYZz-n"
   },
   "outputs": [],
   "source": [
    "gridsearch_params = [\n",
    "    (max_depth, min_child_weight)\n",
    "    for max_depth in range(6,13,1)\n",
    "    for min_child_weight in range(4,41,2)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2654784,
     "status": "ok",
     "timestamp": 1621014799001,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "17Hrqj4TZz-n",
    "outputId": "0a0684bc-0df8-4445-e771-e8a953488aa1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV with max_depth=6, min_child_weight=4\n",
      "\tF1 0.2101268 for 96 rounds\n",
      "CV with max_depth=6, min_child_weight=6\n",
      "\tF1 0.21083980000000002 for 124 rounds\n",
      "CV with max_depth=6, min_child_weight=8\n",
      "\tF1 0.19949160000000002 for 61 rounds\n",
      "CV with max_depth=6, min_child_weight=10\n",
      "\tF1 0.2174718 for 101 rounds\n",
      "CV with max_depth=6, min_child_weight=12\n",
      "\tF1 0.2244386 for 110 rounds\n",
      "CV with max_depth=6, min_child_weight=14\n",
      "\tF1 0.2052088 for 64 rounds\n",
      "CV with max_depth=6, min_child_weight=16\n",
      "\tF1 0.1985248 for 55 rounds\n",
      "CV with max_depth=6, min_child_weight=18\n",
      "\tF1 0.21007820000000002 for 82 rounds\n",
      "CV with max_depth=6, min_child_weight=20\n",
      "\tF1 0.2252976 for 95 rounds\n",
      "CV with max_depth=6, min_child_weight=22\n",
      "\tF1 0.2201486 for 111 rounds\n",
      "CV with max_depth=6, min_child_weight=24\n",
      "\tF1 0.21786700000000003 for 107 rounds\n",
      "CV with max_depth=6, min_child_weight=26\n",
      "\tF1 0.23034980000000002 for 150 rounds\n",
      "CV with max_depth=6, min_child_weight=28\n",
      "\tF1 0.2215456 for 116 rounds\n",
      "CV with max_depth=6, min_child_weight=30\n",
      "\tF1 0.21989039999999999 for 120 rounds\n",
      "CV with max_depth=6, min_child_weight=32\n",
      "\tF1 0.2020384 for 87 rounds\n",
      "CV with max_depth=6, min_child_weight=34\n",
      "\tF1 0.18885 for 48 rounds\n",
      "CV with max_depth=6, min_child_weight=36\n",
      "\tF1 0.2174466 for 113 rounds\n",
      "CV with max_depth=6, min_child_weight=38\n",
      "\tF1 0.1985908 for 66 rounds\n",
      "CV with max_depth=6, min_child_weight=40\n",
      "\tF1 0.20712760000000002 for 88 rounds\n",
      "CV with max_depth=7, min_child_weight=4\n",
      "\tF1 0.20495840000000004 for 58 rounds\n",
      "CV with max_depth=7, min_child_weight=6\n",
      "\tF1 0.20114040000000002 for 46 rounds\n",
      "CV with max_depth=7, min_child_weight=8\n",
      "\tF1 0.2074392 for 64 rounds\n",
      "CV with max_depth=7, min_child_weight=10\n",
      "\tF1 0.217138 for 93 rounds\n",
      "CV with max_depth=7, min_child_weight=12\n",
      "\tF1 0.20722420000000003 for 71 rounds\n",
      "CV with max_depth=7, min_child_weight=14\n",
      "\tF1 0.2198534 for 83 rounds\n",
      "CV with max_depth=7, min_child_weight=16\n",
      "\tF1 0.21778599999999998 for 80 rounds\n",
      "CV with max_depth=7, min_child_weight=18\n",
      "\tF1 0.21292020000000003 for 70 rounds\n",
      "CV with max_depth=7, min_child_weight=20\n",
      "\tF1 0.21305900000000003 for 71 rounds\n",
      "CV with max_depth=7, min_child_weight=22\n",
      "\tF1 0.22400760000000003 for 94 rounds\n",
      "CV with max_depth=7, min_child_weight=24\n",
      "\tF1 0.21481840000000002 for 68 rounds\n",
      "CV with max_depth=7, min_child_weight=26\n",
      "\tF1 0.1971358 for 43 rounds\n",
      "CV with max_depth=7, min_child_weight=28\n",
      "\tF1 0.22794319999999998 for 91 rounds\n",
      "CV with max_depth=7, min_child_weight=30\n",
      "\tF1 0.2093902 for 72 rounds\n",
      "CV with max_depth=7, min_child_weight=32\n",
      "\tF1 0.21644619999999998 for 93 rounds\n",
      "CV with max_depth=7, min_child_weight=34\n",
      "\tF1 0.21658260000000001 for 88 rounds\n",
      "CV with max_depth=7, min_child_weight=36\n",
      "\tF1 0.2131032 for 79 rounds\n",
      "CV with max_depth=7, min_child_weight=38\n",
      "\tF1 0.22163060000000004 for 78 rounds\n",
      "CV with max_depth=7, min_child_weight=40\n",
      "\tF1 0.22015360000000003 for 104 rounds\n",
      "CV with max_depth=8, min_child_weight=4\n",
      "\tF1 0.2158334 for 58 rounds\n",
      "CV with max_depth=8, min_child_weight=6\n",
      "\tF1 0.20132739999999996 for 37 rounds\n",
      "CV with max_depth=8, min_child_weight=8\n",
      "\tF1 0.21428879999999997 for 69 rounds\n",
      "CV with max_depth=8, min_child_weight=10\n",
      "\tF1 0.2095282 for 57 rounds\n",
      "CV with max_depth=8, min_child_weight=12\n",
      "\tF1 0.2183042 for 76 rounds\n",
      "CV with max_depth=8, min_child_weight=14\n",
      "\tF1 0.22195840000000003 for 59 rounds\n",
      "CV with max_depth=8, min_child_weight=16\n",
      "\tF1 0.22780460000000002 for 85 rounds\n",
      "CV with max_depth=8, min_child_weight=18\n",
      "\tF1 0.225692 for 64 rounds\n",
      "CV with max_depth=8, min_child_weight=20\n",
      "\tF1 0.2207004 for 61 rounds\n",
      "CV with max_depth=8, min_child_weight=22\n",
      "\tF1 0.22773119999999997 for 71 rounds\n",
      "CV with max_depth=8, min_child_weight=24\n",
      "\tF1 0.21213359999999998 for 77 rounds\n",
      "CV with max_depth=8, min_child_weight=26\n",
      "\tF1 0.21435680000000001 for 53 rounds\n",
      "CV with max_depth=8, min_child_weight=28\n",
      "\tF1 0.2139136 for 62 rounds\n",
      "CV with max_depth=8, min_child_weight=30\n",
      "\tF1 0.215693 for 63 rounds\n",
      "CV with max_depth=8, min_child_weight=32\n",
      "\tF1 0.21863559999999999 for 78 rounds\n",
      "CV with max_depth=8, min_child_weight=34\n",
      "\tF1 0.21536300000000003 for 85 rounds\n",
      "CV with max_depth=8, min_child_weight=36\n",
      "\tF1 0.2175846 for 71 rounds\n",
      "CV with max_depth=8, min_child_weight=38\n",
      "\tF1 0.2057678 for 47 rounds\n",
      "CV with max_depth=8, min_child_weight=40\n",
      "\tF1 0.21008940000000004 for 74 rounds\n",
      "CV with max_depth=9, min_child_weight=4\n",
      "\tF1 0.1927146 for 34 rounds\n",
      "CV with max_depth=9, min_child_weight=6\n",
      "\tF1 0.21200839999999999 for 50 rounds\n",
      "CV with max_depth=9, min_child_weight=8\n",
      "\tF1 0.21468720000000002 for 44 rounds\n",
      "CV with max_depth=9, min_child_weight=10\n",
      "\tF1 0.2113638 for 59 rounds\n",
      "CV with max_depth=9, min_child_weight=12\n",
      "\tF1 0.217245 for 54 rounds\n",
      "CV with max_depth=9, min_child_weight=14\n",
      "\tF1 0.2198672 for 56 rounds\n",
      "CV with max_depth=9, min_child_weight=16\n",
      "\tF1 0.20787460000000002 for 52 rounds\n",
      "CV with max_depth=9, min_child_weight=18\n",
      "\tF1 0.2239872 for 63 rounds\n",
      "CV with max_depth=9, min_child_weight=20\n",
      "\tF1 0.2188784 for 54 rounds\n",
      "CV with max_depth=9, min_child_weight=22\n",
      "\tF1 0.22516979999999998 for 51 rounds\n",
      "CV with max_depth=9, min_child_weight=24\n",
      "\tF1 0.2248628 for 63 rounds\n",
      "CV with max_depth=9, min_child_weight=26\n",
      "\tF1 0.2175354 for 58 rounds\n",
      "CV with max_depth=9, min_child_weight=28\n",
      "\tF1 0.2226378 for 55 rounds\n",
      "CV with max_depth=9, min_child_weight=30\n",
      "\tF1 0.22964919999999997 for 71 rounds\n",
      "CV with max_depth=9, min_child_weight=32\n",
      "\tF1 0.22577480000000003 for 65 rounds\n",
      "CV with max_depth=9, min_child_weight=34\n",
      "\tF1 0.22365820000000003 for 75 rounds\n",
      "CV with max_depth=9, min_child_weight=36\n",
      "\tF1 0.21307740000000003 for 47 rounds\n",
      "CV with max_depth=9, min_child_weight=38\n",
      "\tF1 0.2220526 for 84 rounds\n",
      "CV with max_depth=9, min_child_weight=40\n",
      "\tF1 0.22326839999999998 for 74 rounds\n",
      "CV with max_depth=10, min_child_weight=4\n",
      "\tF1 0.22954460000000002 for 51 rounds\n",
      "CV with max_depth=10, min_child_weight=6\n",
      "\tF1 0.20580400000000001 for 32 rounds\n",
      "CV with max_depth=10, min_child_weight=8\n",
      "\tF1 0.21426679999999995 for 38 rounds\n",
      "CV with max_depth=10, min_child_weight=10\n",
      "\tF1 0.2140586 for 44 rounds\n",
      "CV with max_depth=10, min_child_weight=12\n",
      "\tF1 0.2363702 for 71 rounds\n",
      "CV with max_depth=10, min_child_weight=14\n",
      "\tF1 0.22629740000000004 for 50 rounds\n",
      "CV with max_depth=10, min_child_weight=16\n",
      "\tF1 0.2118998 for 38 rounds\n",
      "CV with max_depth=10, min_child_weight=18\n",
      "\tF1 0.21213920000000003 for 36 rounds\n",
      "CV with max_depth=10, min_child_weight=20\n",
      "\tF1 0.2369604 for 63 rounds\n",
      "CV with max_depth=10, min_child_weight=22\n",
      "\tF1 0.2156356 for 41 rounds\n",
      "CV with max_depth=10, min_child_weight=24\n",
      "\tF1 0.2277078 for 61 rounds\n",
      "CV with max_depth=10, min_child_weight=26\n",
      "\tF1 0.22337440000000003 for 71 rounds\n",
      "CV with max_depth=10, min_child_weight=28\n",
      "\tF1 0.2102268 for 42 rounds\n",
      "CV with max_depth=10, min_child_weight=30\n",
      "\tF1 0.2180192 for 50 rounds\n",
      "CV with max_depth=10, min_child_weight=32\n",
      "\tF1 0.2194884 for 66 rounds\n",
      "CV with max_depth=10, min_child_weight=34\n",
      "\tF1 0.21917500000000004 for 66 rounds\n",
      "CV with max_depth=10, min_child_weight=36\n",
      "\tF1 0.21604460000000003 for 56 rounds\n",
      "CV with max_depth=10, min_child_weight=38\n",
      "\tF1 0.22013600000000003 for 48 rounds\n",
      "CV with max_depth=10, min_child_weight=40\n",
      "\tF1 0.2231226 for 64 rounds\n",
      "CV with max_depth=11, min_child_weight=4\n",
      "\tF1 0.1959748 for 29 rounds\n",
      "CV with max_depth=11, min_child_weight=6\n",
      "\tF1 0.2149308 for 42 rounds\n",
      "CV with max_depth=11, min_child_weight=8\n",
      "\tF1 0.2217706 for 43 rounds\n",
      "CV with max_depth=11, min_child_weight=10\n",
      "\tF1 0.22005939999999996 for 41 rounds\n",
      "CV with max_depth=11, min_child_weight=12\n",
      "\tF1 0.22525420000000002 for 41 rounds\n",
      "CV with max_depth=11, min_child_weight=14\n",
      "\tF1 0.2218874 for 46 rounds\n",
      "CV with max_depth=11, min_child_weight=16\n",
      "\tF1 0.21744460000000002 for 38 rounds\n",
      "CV with max_depth=11, min_child_weight=18\n",
      "\tF1 0.226619 for 49 rounds\n",
      "CV with max_depth=11, min_child_weight=20\n",
      "\tF1 0.22543100000000002 for 49 rounds\n",
      "CV with max_depth=11, min_child_weight=22\n",
      "\tF1 0.2165426 for 48 rounds\n",
      "CV with max_depth=11, min_child_weight=24\n",
      "\tF1 0.20822639999999998 for 37 rounds\n",
      "CV with max_depth=11, min_child_weight=26\n",
      "\tF1 0.2079098 for 45 rounds\n",
      "CV with max_depth=11, min_child_weight=28\n",
      "\tF1 0.232512 for 86 rounds\n",
      "CV with max_depth=11, min_child_weight=30\n",
      "\tF1 0.2161508 for 44 rounds\n",
      "CV with max_depth=11, min_child_weight=32\n",
      "\tF1 0.2252478 for 64 rounds\n",
      "CV with max_depth=11, min_child_weight=34\n",
      "\tF1 0.21449400000000002 for 47 rounds\n",
      "CV with max_depth=11, min_child_weight=36\n",
      "\tF1 0.21850920000000001 for 36 rounds\n",
      "CV with max_depth=11, min_child_weight=38\n",
      "\tF1 0.227548 for 54 rounds\n",
      "CV with max_depth=11, min_child_weight=40\n",
      "\tF1 0.2313422 for 74 rounds\n",
      "CV with max_depth=12, min_child_weight=4\n",
      "\tF1 0.19361699999999998 for 35 rounds\n",
      "CV with max_depth=12, min_child_weight=6\n",
      "\tF1 0.2084664 for 30 rounds\n",
      "CV with max_depth=12, min_child_weight=8\n",
      "\tF1 0.21345 for 37 rounds\n",
      "CV with max_depth=12, min_child_weight=10\n",
      "\tF1 0.2164622 for 31 rounds\n",
      "CV with max_depth=12, min_child_weight=12\n",
      "\tF1 0.21861379999999997 for 39 rounds\n",
      "CV with max_depth=12, min_child_weight=14\n",
      "\tF1 0.2173476 for 41 rounds\n",
      "CV with max_depth=12, min_child_weight=16\n",
      "\tF1 0.2242338 for 32 rounds\n",
      "CV with max_depth=12, min_child_weight=18\n",
      "\tF1 0.2210226 for 36 rounds\n",
      "CV with max_depth=12, min_child_weight=20\n",
      "\tF1 0.222103 for 50 rounds\n",
      "CV with max_depth=12, min_child_weight=22\n",
      "\tF1 0.22114479999999997 for 44 rounds\n",
      "CV with max_depth=12, min_child_weight=24\n",
      "\tF1 0.227208 for 46 rounds\n",
      "CV with max_depth=12, min_child_weight=26\n",
      "\tF1 0.21755199999999997 for 29 rounds\n",
      "CV with max_depth=12, min_child_weight=28\n",
      "\tF1 0.2271224 for 53 rounds\n",
      "CV with max_depth=12, min_child_weight=30\n",
      "\tF1 0.22719360000000002 for 47 rounds\n",
      "CV with max_depth=12, min_child_weight=32\n",
      "\tF1 0.23420580000000002 for 68 rounds\n",
      "CV with max_depth=12, min_child_weight=34\n",
      "\tF1 0.22398379999999998 for 57 rounds\n",
      "CV with max_depth=12, min_child_weight=36\n",
      "\tF1 0.21704819999999997 for 56 rounds\n",
      "CV with max_depth=12, min_child_weight=38\n",
      "\tF1 0.22252740000000001 for 71 rounds\n",
      "CV with max_depth=12, min_child_weight=40\n",
      "\tF1 0.2240018 for 48 rounds\n",
      "Best params: 10, 20, F1: 0.2369604\n",
      "time: 2654.2538361549377\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "max_f1 = float(0)\n",
    "best_params = None\n",
    "for max_depth, min_child_weight in gridsearch_params:\n",
    "    print(\"CV with max_depth={}, min_child_weight={}\".format(\n",
    "                             max_depth,\n",
    "                             min_child_weight))\n",
    "    \n",
    "    params['max_depth'] = max_depth\n",
    "    params['min_child_weight'] = min_child_weight\n",
    "    \n",
    "    cv_results = xgboost.cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        feval=f1_eval,\n",
    "        early_stopping_rounds = 10,\n",
    "        maximize=True)\n",
    "    \n",
    "    mean_f1 = cv_results['test-F1_score-mean'].max()\n",
    "    boost_rounds = cv_results['test-F1_score-mean'].argmax()\n",
    "    print(\"\\tF1 {} for {} rounds\".format(mean_f1, boost_rounds))\n",
    "    if mean_f1 > max_f1:\n",
    "        max_f1 = mean_f1\n",
    "        best_params = (max_depth,min_child_weight)\n",
    "print(\"Best params: {}, {}, F1: {}\".format(best_params[0], best_params[1], max_f1))\n",
    "end_time = time.time()\n",
    "print('time:', end_time - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "executionInfo": {
     "elapsed": 2649478,
     "status": "ok",
     "timestamp": 1621014799004,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "B-tT0pyOZz-r"
   },
   "outputs": [],
   "source": [
    "params['max_depth'] = best_params[0]\n",
    "params['min_child_weight'] = best_params[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "executionInfo": {
     "elapsed": 2647981,
     "status": "ok",
     "timestamp": 1621014799005,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Kp3Fj6dsZz-s"
   },
   "outputs": [],
   "source": [
    "gridsearch_params = [\n",
    "    (subsample, colsample)\n",
    "    for subsample in [i/10. for i in range(6,11,2)]\n",
    "    for colsample in [i/10. for i in range(4,11,2)]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 939368,
     "status": "ok",
     "timestamp": 1621015068811,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "H71Y1EsPZz-s",
    "outputId": "5e3a303b-686f-4fb9-9ffa-638db9f6455c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV with subsample=1.0, colsample=1.0\n",
      "\tF1 0.2369604 for 63 rounds\n",
      "CV with subsample=1.0, colsample=0.8\n",
      "\tF1 0.21881659999999997 for 66 rounds\n",
      "CV with subsample=1.0, colsample=0.6\n",
      "\tF1 0.22173099999999998 for 72 rounds\n",
      "CV with subsample=1.0, colsample=0.4\n",
      "\tF1 0.23385799999999998 for 73 rounds\n",
      "CV with subsample=0.8, colsample=1.0\n",
      "\tF1 0.2181352 for 47 rounds\n",
      "CV with subsample=0.8, colsample=0.8\n",
      "\tF1 0.2248684 for 78 rounds\n",
      "CV with subsample=0.8, colsample=0.6\n",
      "\tF1 0.20538240000000002 for 29 rounds\n",
      "CV with subsample=0.8, colsample=0.4\n",
      "\tF1 0.2196986 for 68 rounds\n",
      "CV with subsample=0.6, colsample=1.0\n",
      "\tF1 0.21357379999999998 for 64 rounds\n",
      "CV with subsample=0.6, colsample=0.8\n",
      "\tF1 0.198546 for 37 rounds\n",
      "CV with subsample=0.6, colsample=0.6\n",
      "\tF1 0.20552739999999997 for 49 rounds\n",
      "CV with subsample=0.6, colsample=0.4\n",
      "\tF1 0.20166119999999998 for 49 rounds\n",
      "Best params: 1.0, 1.0, F1: 0.2369604\n",
      "time: 269.4898133277893\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "max_f1 = float(0)\n",
    "best_params = None\n",
    "\n",
    "for subsample, colsample in reversed(gridsearch_params):\n",
    "    print(\"CV with subsample={}, colsample={}\".format(\n",
    "                             subsample,\n",
    "                             colsample))\n",
    "    \n",
    "    params['subsample'] = subsample\n",
    "    params['colsample_bytree'] = colsample\n",
    "    \n",
    "    cv_results = xgboost.cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        feval=f1_eval,\n",
    "        early_stopping_rounds = 10,\n",
    "        maximize=True)\n",
    "    \n",
    "    mean_f1 = cv_results['test-F1_score-mean'].max()\n",
    "    boost_rounds = cv_results['test-F1_score-mean'].argmax()\n",
    "    print(\"\\tF1 {} for {} rounds\".format(mean_f1, boost_rounds))\n",
    "    if mean_f1 > max_f1:\n",
    "        max_f1 = mean_f1\n",
    "        best_params = (subsample,colsample)\n",
    "print(\"Best params: {}, {}, F1: {}\".format(best_params[0], best_params[1], max_f1))\n",
    "end_time = time.time()\n",
    "print('time:', end_time - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "executionInfo": {
     "elapsed": 636,
     "status": "ok",
     "timestamp": 1621015069471,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Gtyy6RneZz-t"
   },
   "outputs": [],
   "source": [
    "params['subsample'] = best_params[0]\n",
    "params['colsample_bytree'] = best_params[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "executionInfo": {
     "elapsed": 78618,
     "status": "ok",
     "timestamp": 1621015147473,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "5-yRvw_xZz-t",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "eb781367-cce5-4592-fe4e-e95712b91c9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV with eta=0.3\n",
      "\tF1 0.2369604 for 63 rounds\n",
      "\n",
      "CV with eta=0.2\n",
      "\tF1 0.2158298 for 47 rounds\n",
      "\n",
      "CV with eta=0.1\n",
      "\tF1 0.21139420000000003 for 80 rounds\n",
      "\n",
      "CV with eta=0.01\n",
      "\tF1 0.1259794 for 3 rounds\n",
      "\n",
      "CV with eta=0.005\n",
      "\tF1 0.12478040000000001 for 7 rounds\n",
      "\n",
      "Best params: 0.3, F1: 0.2369604\n",
      "time: 77.85641241073608\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "max_f1 = float(0)\n",
    "best_params = None\n",
    "\n",
    "for eta in [0.3, 0.2, 0.1, 0.01, 0.005]:\n",
    "    print(\"CV with eta={}\".format(eta))\n",
    "\n",
    "    params['eta'] = eta\n",
    "\n",
    "    cv_results = xgboost.cv(\n",
    "            params,\n",
    "            dtrain,\n",
    "            num_boost_round=num_boost_round,\n",
    "            seed=42,\n",
    "            nfold=5,\n",
    "            feval=f1_eval,\n",
    "            early_stopping_rounds = 10,\n",
    "            maximize=True)\n",
    "\n",
    "    mean_f1 = cv_results['test-F1_score-mean'].max()\n",
    "    boost_rounds = cv_results['test-F1_score-mean'].argmax()\n",
    "    print(\"\\tF1 {} for {} rounds\\n\".format(mean_f1, boost_rounds))\n",
    "    if mean_f1 > max_f1:\n",
    "        max_f1 = mean_f1\n",
    "        best_params = eta\n",
    "print(\"Best params: {}, F1: {}\".format(best_params, max_f1))\n",
    "end_time = time.time()\n",
    "print('time:', end_time - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "executionInfo": {
     "elapsed": 78617,
     "status": "ok",
     "timestamp": 1621015147474,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Lcq7bcJZZz-t"
   },
   "outputs": [],
   "source": [
    "params['eta'] = best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fgg84yHVZz-u"
   },
   "source": [
    "### 3.2 Final model training and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "executionInfo": {
     "elapsed": 777,
     "status": "ok",
     "timestamp": 1621016167240,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "E395ExzPZz-u"
   },
   "outputs": [],
   "source": [
    "# optimized hyper-parameters\n",
    "params = {'max_depth': 10,\n",
    " 'min_child_weight': 20,\n",
    " 'eta': 0.3,\n",
    " 'subsample': 1.0,\n",
    " 'colsample_bytree': 1.0,\n",
    " 'objective': 'binary:logistic',\n",
    " 'disable_default_eval_metric': 1,\n",
    " 'seed': 42,\n",
    " 'tree_method': 'hist',\n",
    " 'scale_pos_weight': scale_pos_weight}\n",
    "num_boost_round=999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13046,
     "status": "ok",
     "timestamp": 1621016179516,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "qKJXTfeuZz-u",
    "outputId": "8ef38627-76d0-44fa-95ae-5d7fd507a400",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.17689\tdval-F1_score:0.154396\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.192269\tdval-F1_score:0.16851\n",
      "[2]\tdtrain-F1_score:0.204314\tdval-F1_score:0.174334\n",
      "[3]\tdtrain-F1_score:0.214406\tdval-F1_score:0.17945\n",
      "[4]\tdtrain-F1_score:0.221137\tdval-F1_score:0.183658\n",
      "[5]\tdtrain-F1_score:0.22793\tdval-F1_score:0.187067\n",
      "[6]\tdtrain-F1_score:0.237969\tdval-F1_score:0.193626\n",
      "[7]\tdtrain-F1_score:0.243109\tdval-F1_score:0.195581\n",
      "[8]\tdtrain-F1_score:0.250195\tdval-F1_score:0.201098\n",
      "[9]\tdtrain-F1_score:0.252304\tdval-F1_score:0.202542\n",
      "[10]\tdtrain-F1_score:0.255241\tdval-F1_score:0.202899\n",
      "[11]\tdtrain-F1_score:0.261164\tdval-F1_score:0.209059\n",
      "[12]\tdtrain-F1_score:0.268079\tdval-F1_score:0.214095\n",
      "[13]\tdtrain-F1_score:0.271151\tdval-F1_score:0.213189\n",
      "[14]\tdtrain-F1_score:0.274423\tdval-F1_score:0.213272\n",
      "[15]\tdtrain-F1_score:0.277637\tdval-F1_score:0.216266\n",
      "[16]\tdtrain-F1_score:0.28237\tdval-F1_score:0.215428\n",
      "[17]\tdtrain-F1_score:0.284547\tdval-F1_score:0.215064\n",
      "[18]\tdtrain-F1_score:0.28707\tdval-F1_score:0.218182\n",
      "[19]\tdtrain-F1_score:0.289482\tdval-F1_score:0.219336\n",
      "[20]\tdtrain-F1_score:0.292003\tdval-F1_score:0.220823\n",
      "[21]\tdtrain-F1_score:0.295284\tdval-F1_score:0.222877\n",
      "[22]\tdtrain-F1_score:0.300775\tdval-F1_score:0.223214\n",
      "[23]\tdtrain-F1_score:0.306556\tdval-F1_score:0.224112\n",
      "[24]\tdtrain-F1_score:0.312094\tdval-F1_score:0.229391\n",
      "[25]\tdtrain-F1_score:0.314503\tdval-F1_score:0.229626\n",
      "[26]\tdtrain-F1_score:0.318046\tdval-F1_score:0.229016\n",
      "[27]\tdtrain-F1_score:0.323841\tdval-F1_score:0.233316\n",
      "[28]\tdtrain-F1_score:0.32721\tdval-F1_score:0.235232\n",
      "[29]\tdtrain-F1_score:0.329311\tdval-F1_score:0.234099\n",
      "[30]\tdtrain-F1_score:0.331776\tdval-F1_score:0.234979\n",
      "[31]\tdtrain-F1_score:0.333333\tdval-F1_score:0.23402\n",
      "[32]\tdtrain-F1_score:0.336146\tdval-F1_score:0.234655\n",
      "[33]\tdtrain-F1_score:0.342539\tdval-F1_score:0.238938\n",
      "[34]\tdtrain-F1_score:0.34555\tdval-F1_score:0.239155\n",
      "[35]\tdtrain-F1_score:0.348236\tdval-F1_score:0.238388\n",
      "[36]\tdtrain-F1_score:0.35068\tdval-F1_score:0.238873\n",
      "[37]\tdtrain-F1_score:0.351904\tdval-F1_score:0.238553\n",
      "[38]\tdtrain-F1_score:0.354649\tdval-F1_score:0.240868\n",
      "[39]\tdtrain-F1_score:0.357645\tdval-F1_score:0.244662\n",
      "[40]\tdtrain-F1_score:0.364281\tdval-F1_score:0.248366\n",
      "[41]\tdtrain-F1_score:0.367317\tdval-F1_score:0.251353\n",
      "[42]\tdtrain-F1_score:0.369444\tdval-F1_score:0.251051\n",
      "[43]\tdtrain-F1_score:0.372318\tdval-F1_score:0.254391\n",
      "[44]\tdtrain-F1_score:0.373344\tdval-F1_score:0.256223\n",
      "[45]\tdtrain-F1_score:0.376024\tdval-F1_score:0.25\n",
      "[46]\tdtrain-F1_score:0.377251\tdval-F1_score:0.250776\n",
      "[47]\tdtrain-F1_score:0.37991\tdval-F1_score:0.250936\n",
      "[48]\tdtrain-F1_score:0.381607\tdval-F1_score:0.25078\n",
      "[49]\tdtrain-F1_score:0.383617\tdval-F1_score:0.251889\n",
      "[50]\tdtrain-F1_score:0.387092\tdval-F1_score:0.256051\n",
      "[51]\tdtrain-F1_score:0.396295\tdval-F1_score:0.256976\n",
      "[52]\tdtrain-F1_score:0.4\tdval-F1_score:0.262467\n",
      "[53]\tdtrain-F1_score:0.40316\tdval-F1_score:0.262014\n",
      "[54]\tdtrain-F1_score:0.405719\tdval-F1_score:0.260754\n",
      "[55]\tdtrain-F1_score:0.407652\tdval-F1_score:0.260754\n",
      "[56]\tdtrain-F1_score:0.410298\tdval-F1_score:0.262735\n",
      "[57]\tdtrain-F1_score:0.414142\tdval-F1_score:0.264865\n",
      "[58]\tdtrain-F1_score:0.41822\tdval-F1_score:0.263692\n",
      "[59]\tdtrain-F1_score:0.423118\tdval-F1_score:0.261823\n",
      "[60]\tdtrain-F1_score:0.425067\tdval-F1_score:0.263812\n",
      "[61]\tdtrain-F1_score:0.430129\tdval-F1_score:0.267596\n",
      "[62]\tdtrain-F1_score:0.433043\tdval-F1_score:0.270119\n",
      "[63]\tdtrain-F1_score:0.437505\tdval-F1_score:0.269394\n",
      "[64]\tdtrain-F1_score:0.438056\tdval-F1_score:0.271114\n",
      "[65]\tdtrain-F1_score:0.440783\tdval-F1_score:0.268763\n",
      "[66]\tdtrain-F1_score:0.443665\tdval-F1_score:0.26705\n",
      "[67]\tdtrain-F1_score:0.447123\tdval-F1_score:0.269258\n",
      "[68]\tdtrain-F1_score:0.449144\tdval-F1_score:0.267534\n",
      "[69]\tdtrain-F1_score:0.450782\tdval-F1_score:0.268505\n",
      "[70]\tdtrain-F1_score:0.452027\tdval-F1_score:0.268417\n",
      "[71]\tdtrain-F1_score:0.452836\tdval-F1_score:0.27193\n",
      "[72]\tdtrain-F1_score:0.45644\tdval-F1_score:0.268722\n",
      "[73]\tdtrain-F1_score:0.458055\tdval-F1_score:0.268328\n",
      "[74]\tdtrain-F1_score:0.460482\tdval-F1_score:0.272459\n",
      "[75]\tdtrain-F1_score:0.462357\tdval-F1_score:0.273871\n",
      "[76]\tdtrain-F1_score:0.466583\tdval-F1_score:0.271111\n",
      "[77]\tdtrain-F1_score:0.468587\tdval-F1_score:0.271312\n",
      "[78]\tdtrain-F1_score:0.471605\tdval-F1_score:0.274569\n",
      "[79]\tdtrain-F1_score:0.475637\tdval-F1_score:0.276386\n",
      "[80]\tdtrain-F1_score:0.479201\tdval-F1_score:0.277863\n",
      "[81]\tdtrain-F1_score:0.482693\tdval-F1_score:0.277395\n",
      "[82]\tdtrain-F1_score:0.485388\tdval-F1_score:0.280431\n",
      "[83]\tdtrain-F1_score:0.488756\tdval-F1_score:0.281911\n",
      "[84]\tdtrain-F1_score:0.492259\tdval-F1_score:0.283902\n",
      "[85]\tdtrain-F1_score:0.494095\tdval-F1_score:0.282989\n",
      "[86]\tdtrain-F1_score:0.497274\tdval-F1_score:0.282989\n",
      "[87]\tdtrain-F1_score:0.49879\tdval-F1_score:0.286853\n",
      "[88]\tdtrain-F1_score:0.50063\tdval-F1_score:0.285032\n",
      "[89]\tdtrain-F1_score:0.502302\tdval-F1_score:0.2848\n",
      "[90]\tdtrain-F1_score:0.507201\tdval-F1_score:0.290323\n",
      "[91]\tdtrain-F1_score:0.50836\tdval-F1_score:0.290557\n",
      "[92]\tdtrain-F1_score:0.510599\tdval-F1_score:0.289879\n",
      "[93]\tdtrain-F1_score:0.512433\tdval-F1_score:0.29316\n",
      "[94]\tdtrain-F1_score:0.513711\tdval-F1_score:0.29085\n",
      "[95]\tdtrain-F1_score:0.514899\tdval-F1_score:0.289689\n",
      "[96]\tdtrain-F1_score:0.515376\tdval-F1_score:0.288288\n",
      "[97]\tdtrain-F1_score:0.51874\tdval-F1_score:0.28642\n",
      "[98]\tdtrain-F1_score:0.520779\tdval-F1_score:0.287129\n",
      "[99]\tdtrain-F1_score:0.522\tdval-F1_score:0.287366\n",
      "[100]\tdtrain-F1_score:0.523178\tdval-F1_score:0.286892\n",
      "[101]\tdtrain-F1_score:0.525352\tdval-F1_score:0.287129\n",
      "[102]\tdtrain-F1_score:0.528998\tdval-F1_score:0.287137\n",
      "[103]\tdtrain-F1_score:0.530511\tdval-F1_score:0.288333\n",
      "Stopping. Best iteration:\n",
      "[93]\tdtrain-F1_score:0.512433\tdval-F1_score:0.29316\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgboost_tuned_weighted = xgboost.train(\n",
    "    params,\n",
    "    dtrain1,\n",
    "    num_boost_round=num_boost_round,\n",
    "    evals=[(dtrain1, 'dtrain'), (dval, 'dval')],\n",
    "    feval=f1_eval,\n",
    "    early_stopping_rounds = 10,\n",
    "    maximize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gS7tSUMUZz-v"
   },
   "outputs": [],
   "source": [
    "# xgboost_tuned_weighted.save_model(\"xgboost_tuned_weighted.model\")\n",
    "# xgboost_tuned_weighted = xgboost.Booster()\n",
    "# xgboost_tuned_weighted.load_model(\"xgboost_tuned_weighted.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 696,
     "status": "ok",
     "timestamp": 1621016180304,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "j-i0l-KWZz-v",
    "outputId": "d7c4952e-fece-4f06-ecbf-53b28db6fbee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of xgboost_tuned_weighted, test:\n",
      "F1: 0.32\n",
      "Precision: 0.22\n",
      "Recall: 0.62\n",
      "AUC: 0.75\n",
      "Accuracy: 0.86\n",
      "Confusion matrix:\n",
      " [[5377  127]\n",
      " [ 750  210]]\n"
     ]
    }
   ],
   "source": [
    "xgboost_tuned_weighted_predt = np.where(xgboost_tuned_weighted.predict(dtest) > 0.5, 1, 0)\n",
    "print('Performance of xgboost_tuned_weighted, test:')\n",
    "print('F1:', round(metrics.f1_score(dtest.get_label(),xgboost_tuned_weighted_predt),2))\n",
    "print('Precision:', round(metrics.precision_score(dtest.get_label(), xgboost_tuned_weighted_predt),2))\n",
    "print('Recall:', round(metrics.recall_score(dtest.get_label(), xgboost_tuned_weighted_predt),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(dtest.get_label(), xgboost_tuned_weighted_predt),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(dtest.get_label(), xgboost_tuned_weighted_predt),2))\n",
    "print('Confusion matrix:\\n', \n",
    "      metrics.confusion_matrix(dtest.get_label(), xgboost_tuned_weighted_predt).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1151,
     "status": "ok",
     "timestamp": 1621016180768,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "01YDAjrznc8S",
    "outputId": "e61076a8-230d-4c7b-c786-56a5ba1b651f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "test_sentences['pred'] = xgboost_tuned_weighted_predt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1144,
     "status": "ok",
     "timestamp": 1621016180770,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Yeo1Oq0UOY1k",
    "outputId": "2bdacd97-d6be-4491-9182-cbc0d3ee0663"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2957"
      ]
     },
     "execution_count": 58,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_sentences['sentence'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 599,
     "status": "ok",
     "timestamp": 1621016181384,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Lwu2c3W1NSvN",
    "outputId": "6077314a-c461-4a19-fc88-c4a2ccad68b7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of predicted biased words per sentence:\n",
      " 0    2122\n",
      "1     723\n",
      "2     100\n",
      "3      11\n",
      "4       1\n",
      "Name: pred, dtype: int64\n",
      "Prediction on the sentence level:\n",
      " 0    2122\n",
      "1     835\n",
      "Name: Label_bias_hat, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# count how many words per sentence are predicted as biased\n",
    "test_sentences_hat = test_sentences.groupby('sentence', as_index=False).agg({'pred': 'sum'})\n",
    "print('Number of predicted biased words per sentence:\\n',test_sentences_hat['pred'].value_counts())\n",
    "\n",
    "# rule: if one ore more bias words are in a sentence, the sentence will be labelled as biased\n",
    "test_sentences_hat['Label_bias_hat'] = test_sentences_hat['pred'].apply(lambda x: 1 if x > 0 else 0)\n",
    "print('Prediction on the sentence level:\\n', test_sentences_hat['Label_bias_hat'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 672
    },
    "executionInfo": {
     "elapsed": 1365,
     "status": "ok",
     "timestamp": 1621016182163,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "fb3fWP5drJyd",
    "outputId": "b92ef700-1ee1-4f42-c3b6-6cfce60f00ef"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>pred</th>\n",
       "      <th>Label_bias_hat</th>\n",
       "      <th>news_link</th>\n",
       "      <th>outlet</th>\n",
       "      <th>topic</th>\n",
       "      <th>type</th>\n",
       "      <th>Label_bias</th>\n",
       "      <th>label_opinion</th>\n",
       "      <th>biased_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'Pandemic Apprentice': Deprived of rallies to ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.alternet.org/2020/04/pandemic-appr...</td>\n",
       "      <td>Alternet</td>\n",
       "      <td>gun control</td>\n",
       "      <td>left</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Somewhat factual but also opinionated</td>\n",
       "      <td>['ego', 'muscles', 'in']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>...immigrants as criminals and eugenics, all o...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.nbcnews.com/news/latino/after-step...</td>\n",
       "      <td>MSNBC</td>\n",
       "      <td>white-nationalism</td>\n",
       "      <td>left</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Expresses writer’s opinion</td>\n",
       "      <td>['criminals', 'fringe', 'extreme']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>...we sounded the alarm in the early months of...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.alternet.org/2019/07/fox-news-has-...</td>\n",
       "      <td>Alternet</td>\n",
       "      <td>white-nationalism</td>\n",
       "      <td>left</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Somewhat factual but also opinionated</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A 10-hour hearing broadcast on the court’s web...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.foxnews.com/sports/olympic-swimmin...</td>\n",
       "      <td>Fox News</td>\n",
       "      <td>sport</td>\n",
       "      <td>right</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Somewhat factual but also opinionated</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A 16-time NBA All Star who is considered one o...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.reuters.com/article/us-usa-electio...</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>black lives matter</td>\n",
       "      <td>center</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Entirely factual</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2952</th>\n",
       "      <td>“He won because the Election was Rigged,” Trum...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.reuters.com/article/us-usa-electio...</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>vaccine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Entirely factual</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2953</th>\n",
       "      <td>“I would shut it down; I would listen to the s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://thefederalist.com/2020/08/22/biden-i-w...</td>\n",
       "      <td>Federalist</td>\n",
       "      <td>vaccine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Entirely factual</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2954</th>\n",
       "      <td>“Mike Pence and the task force have done a gre...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.reuters.com/article/us-health-coro...</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>vaccine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Entirely factual</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2955</th>\n",
       "      <td>“She will be inheriting a government in crisis...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.breitbart.com/politics/2020/01/23/...</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>marriage-equality</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Somewhat factual but also opinionated</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2956</th>\n",
       "      <td>“The goal is to send a message of peace,” the ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>http://feedproxy.google.com/~r/breitbart/~3/Ke...</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>black lives matter</td>\n",
       "      <td>right</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Somewhat factual but also opinionated</td>\n",
       "      <td>['claimed']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2956 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence  ...                        biased_words\n",
       "0     'Pandemic Apprentice': Deprived of rallies to ...  ...            ['ego', 'muscles', 'in']\n",
       "1     ...immigrants as criminals and eugenics, all o...  ...  ['criminals', 'fringe', 'extreme']\n",
       "2     ...we sounded the alarm in the early months of...  ...                                  []\n",
       "3     A 10-hour hearing broadcast on the court’s web...  ...                                  []\n",
       "4     A 16-time NBA All Star who is considered one o...  ...                                  []\n",
       "...                                                 ...  ...                                 ...\n",
       "2952  “He won because the Election was Rigged,” Trum...  ...                                  []\n",
       "2953  “I would shut it down; I would listen to the s...  ...                                  []\n",
       "2954  “Mike Pence and the task force have done a gre...  ...                                  []\n",
       "2955  “She will be inheriting a government in crisis...  ...                                  []\n",
       "2956  “The goal is to send a message of peace,” the ...  ...                         ['claimed']\n",
       "\n",
       "[2956 rows x 10 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load ground-truth labels on the sentence level\n",
    "#PATH_sg1 = \"data/final_labels_SG1.xlsx\"\n",
    "PATH_sg2 = \"data/final_labels_SG2.xlsx\"\n",
    "#sentences_ground_truth = pd.read_excel(PATH_sg1)\n",
    "sentences_ground_truth = pd.read_excel(PATH_sg2)\n",
    "sentences_ground_truth.rename(columns={'text': 'sentence', 'label_bias': 'Label_bias'}, inplace=True)\n",
    "sentences_ground_truth = sentences_ground_truth[sentences_ground_truth['Label_bias']!='No agreement']\n",
    "sentences_ground_truth.replace(to_replace='Biased', value=1, inplace=True)\n",
    "sentences_ground_truth.replace(to_replace='Non-biased', value=0, inplace=True)\n",
    "\n",
    "sentences_ground_truth = test_sentences_hat.merge(right=sentences_ground_truth, how='left', on='sentence')\n",
    "sentences_ground_truth = sentences_ground_truth[sentences_ground_truth['Label_bias'].isna() == False]\n",
    "sentences_ground_truth "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 740,
     "status": "ok",
     "timestamp": 1621016182932,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "_2NxFgEjtuLC",
    "outputId": "1795a0f0-9365-4236-fda8-89630219f5f4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance of xgboost_tuned_weighted, test:\n",
      "F1: 0.45\n",
      "Precision: 0.61\n",
      "Recall: 0.35\n",
      "AUC: 0.57\n",
      "Accuracy: 0.57\n",
      "Confusion matrix:\n",
      " [[1192  929]\n",
      " [ 329  506]]\n"
     ]
    }
   ],
   "source": [
    "# compute performance\n",
    "print('Performance of xgboost_tuned_weighted, test:')\n",
    "print('F1:', round(metrics.f1_score(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']),2))\n",
    "print('Precision:', round(metrics.precision_score(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']),2))\n",
    "print('Recall:', round(metrics.recall_score(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']),2))\n",
    "print('AUC:', round(metrics.roc_auc_score(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']),2))\n",
    "print('Accuracy:', round(metrics.accuracy_score(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']),2))\n",
    "print('Confusion matrix:\\n', \n",
    "      metrics.confusion_matrix(sentences_ground_truth['Label_bias'],sentences_ground_truth['Label_bias_hat']).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 79328,
     "status": "ok",
     "timestamp": 1621016261528,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "ewENvVtvUQ-H",
    "outputId": "ed0b7abf-08eb-4966-c637-ee1bbbabad4f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.185492\tdval-F1_score:0.174846\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.199749\tdval-F1_score:0.185264\n",
      "[2]\tdtrain-F1_score:0.223744\tdval-F1_score:0.205913\n",
      "[3]\tdtrain-F1_score:0.212149\tdval-F1_score:0.193831\n",
      "[4]\tdtrain-F1_score:0.228156\tdval-F1_score:0.206642\n",
      "[5]\tdtrain-F1_score:0.239009\tdval-F1_score:0.21365\n",
      "[6]\tdtrain-F1_score:0.237706\tdval-F1_score:0.209025\n",
      "[7]\tdtrain-F1_score:0.241821\tdval-F1_score:0.211009\n",
      "[8]\tdtrain-F1_score:0.247102\tdval-F1_score:0.213636\n",
      "[9]\tdtrain-F1_score:0.251\tdval-F1_score:0.216532\n",
      "[10]\tdtrain-F1_score:0.257378\tdval-F1_score:0.220201\n",
      "[11]\tdtrain-F1_score:0.260933\tdval-F1_score:0.220479\n",
      "[12]\tdtrain-F1_score:0.265238\tdval-F1_score:0.224179\n",
      "[13]\tdtrain-F1_score:0.268141\tdval-F1_score:0.226126\n",
      "[14]\tdtrain-F1_score:0.270988\tdval-F1_score:0.227931\n",
      "[15]\tdtrain-F1_score:0.273575\tdval-F1_score:0.22973\n",
      "[16]\tdtrain-F1_score:0.280325\tdval-F1_score:0.235146\n",
      "[17]\tdtrain-F1_score:0.284973\tdval-F1_score:0.234618\n",
      "[18]\tdtrain-F1_score:0.2885\tdval-F1_score:0.236305\n",
      "[19]\tdtrain-F1_score:0.293075\tdval-F1_score:0.239424\n",
      "[20]\tdtrain-F1_score:0.295745\tdval-F1_score:0.240193\n",
      "[21]\tdtrain-F1_score:0.30015\tdval-F1_score:0.240745\n",
      "[22]\tdtrain-F1_score:0.302306\tdval-F1_score:0.241817\n",
      "[23]\tdtrain-F1_score:0.305366\tdval-F1_score:0.244999\n",
      "[24]\tdtrain-F1_score:0.307404\tdval-F1_score:0.246482\n",
      "[25]\tdtrain-F1_score:0.309717\tdval-F1_score:0.248513\n",
      "[26]\tdtrain-F1_score:0.312987\tdval-F1_score:0.247869\n",
      "[27]\tdtrain-F1_score:0.317273\tdval-F1_score:0.252054\n",
      "[28]\tdtrain-F1_score:0.319026\tdval-F1_score:0.253052\n",
      "[29]\tdtrain-F1_score:0.320713\tdval-F1_score:0.252716\n",
      "[30]\tdtrain-F1_score:0.323503\tdval-F1_score:0.253394\n",
      "[31]\tdtrain-F1_score:0.326958\tdval-F1_score:0.255288\n",
      "[32]\tdtrain-F1_score:0.332232\tdval-F1_score:0.257178\n",
      "[33]\tdtrain-F1_score:0.333866\tdval-F1_score:0.258413\n",
      "[34]\tdtrain-F1_score:0.339986\tdval-F1_score:0.261822\n",
      "[35]\tdtrain-F1_score:0.345907\tdval-F1_score:0.263039\n",
      "[36]\tdtrain-F1_score:0.348199\tdval-F1_score:0.263118\n",
      "[37]\tdtrain-F1_score:0.353354\tdval-F1_score:0.265947\n",
      "[38]\tdtrain-F1_score:0.358502\tdval-F1_score:0.266285\n",
      "[39]\tdtrain-F1_score:0.362342\tdval-F1_score:0.267089\n",
      "[40]\tdtrain-F1_score:0.364427\tdval-F1_score:0.268577\n",
      "[41]\tdtrain-F1_score:0.367732\tdval-F1_score:0.268737\n",
      "[42]\tdtrain-F1_score:0.372338\tdval-F1_score:0.270754\n",
      "[43]\tdtrain-F1_score:0.375922\tdval-F1_score:0.27128\n",
      "[44]\tdtrain-F1_score:0.378848\tdval-F1_score:0.272602\n",
      "[45]\tdtrain-F1_score:0.382013\tdval-F1_score:0.273509\n",
      "[46]\tdtrain-F1_score:0.38712\tdval-F1_score:0.276715\n",
      "[47]\tdtrain-F1_score:0.389107\tdval-F1_score:0.278754\n",
      "[48]\tdtrain-F1_score:0.393284\tdval-F1_score:0.281025\n",
      "[49]\tdtrain-F1_score:0.396743\tdval-F1_score:0.282223\n",
      "[50]\tdtrain-F1_score:0.398162\tdval-F1_score:0.283543\n",
      "[51]\tdtrain-F1_score:0.398889\tdval-F1_score:0.284124\n",
      "[52]\tdtrain-F1_score:0.400147\tdval-F1_score:0.283875\n",
      "[53]\tdtrain-F1_score:0.402003\tdval-F1_score:0.285125\n",
      "[54]\tdtrain-F1_score:0.404953\tdval-F1_score:0.286137\n",
      "[55]\tdtrain-F1_score:0.408008\tdval-F1_score:0.289042\n",
      "[56]\tdtrain-F1_score:0.411109\tdval-F1_score:0.290439\n",
      "[57]\tdtrain-F1_score:0.41533\tdval-F1_score:0.292846\n",
      "[58]\tdtrain-F1_score:0.418826\tdval-F1_score:0.29473\n",
      "[59]\tdtrain-F1_score:0.421247\tdval-F1_score:0.294354\n",
      "[60]\tdtrain-F1_score:0.423467\tdval-F1_score:0.293391\n",
      "[61]\tdtrain-F1_score:0.42485\tdval-F1_score:0.295011\n",
      "[62]\tdtrain-F1_score:0.427042\tdval-F1_score:0.298507\n",
      "[63]\tdtrain-F1_score:0.430406\tdval-F1_score:0.296366\n",
      "[64]\tdtrain-F1_score:0.434064\tdval-F1_score:0.296998\n",
      "[65]\tdtrain-F1_score:0.436561\tdval-F1_score:0.298763\n",
      "[66]\tdtrain-F1_score:0.438031\tdval-F1_score:0.298982\n",
      "[67]\tdtrain-F1_score:0.443219\tdval-F1_score:0.3\n",
      "[68]\tdtrain-F1_score:0.444987\tdval-F1_score:0.29987\n",
      "[69]\tdtrain-F1_score:0.448273\tdval-F1_score:0.302083\n",
      "[70]\tdtrain-F1_score:0.451011\tdval-F1_score:0.303525\n",
      "[71]\tdtrain-F1_score:0.453745\tdval-F1_score:0.304962\n",
      "[72]\tdtrain-F1_score:0.453971\tdval-F1_score:0.305163\n",
      "[73]\tdtrain-F1_score:0.459662\tdval-F1_score:0.307692\n",
      "[74]\tdtrain-F1_score:0.463129\tdval-F1_score:0.309973\n",
      "[75]\tdtrain-F1_score:0.465021\tdval-F1_score:0.311276\n",
      "[76]\tdtrain-F1_score:0.465735\tdval-F1_score:0.311126\n",
      "[77]\tdtrain-F1_score:0.467048\tdval-F1_score:0.313926\n",
      "[78]\tdtrain-F1_score:0.469174\tdval-F1_score:0.315251\n",
      "[79]\tdtrain-F1_score:0.472052\tdval-F1_score:0.315139\n",
      "[80]\tdtrain-F1_score:0.47733\tdval-F1_score:0.316811\n",
      "[81]\tdtrain-F1_score:0.481919\tdval-F1_score:0.318039\n",
      "[82]\tdtrain-F1_score:0.483968\tdval-F1_score:0.317159\n",
      "[83]\tdtrain-F1_score:0.486684\tdval-F1_score:0.319689\n",
      "[84]\tdtrain-F1_score:0.488468\tdval-F1_score:0.320481\n",
      "[85]\tdtrain-F1_score:0.489737\tdval-F1_score:0.320654\n",
      "[86]\tdtrain-F1_score:0.492385\tdval-F1_score:0.320114\n",
      "[87]\tdtrain-F1_score:0.493899\tdval-F1_score:0.318312\n",
      "[88]\tdtrain-F1_score:0.498677\tdval-F1_score:0.318264\n",
      "[89]\tdtrain-F1_score:0.501009\tdval-F1_score:0.319419\n",
      "[90]\tdtrain-F1_score:0.503641\tdval-F1_score:0.319942\n",
      "[91]\tdtrain-F1_score:0.505224\tdval-F1_score:0.320616\n",
      "[92]\tdtrain-F1_score:0.5071\tdval-F1_score:0.323128\n",
      "[93]\tdtrain-F1_score:0.508137\tdval-F1_score:0.322986\n",
      "[94]\tdtrain-F1_score:0.511132\tdval-F1_score:0.321098\n",
      "[95]\tdtrain-F1_score:0.515423\tdval-F1_score:0.321977\n",
      "[96]\tdtrain-F1_score:0.518603\tdval-F1_score:0.321589\n",
      "[97]\tdtrain-F1_score:0.520183\tdval-F1_score:0.322775\n",
      "[98]\tdtrain-F1_score:0.524981\tdval-F1_score:0.322311\n",
      "[99]\tdtrain-F1_score:0.526752\tdval-F1_score:0.323294\n",
      "[100]\tdtrain-F1_score:0.528433\tdval-F1_score:0.322877\n",
      "[101]\tdtrain-F1_score:0.531001\tdval-F1_score:0.326279\n",
      "[102]\tdtrain-F1_score:0.532502\tdval-F1_score:0.325492\n",
      "[103]\tdtrain-F1_score:0.533802\tdval-F1_score:0.325886\n",
      "[104]\tdtrain-F1_score:0.535371\tdval-F1_score:0.325869\n",
      "[105]\tdtrain-F1_score:0.536791\tdval-F1_score:0.328149\n",
      "[106]\tdtrain-F1_score:0.537584\tdval-F1_score:0.329045\n",
      "[107]\tdtrain-F1_score:0.541258\tdval-F1_score:0.330572\n",
      "[108]\tdtrain-F1_score:0.545527\tdval-F1_score:0.330821\n",
      "[109]\tdtrain-F1_score:0.546072\tdval-F1_score:0.332538\n",
      "[110]\tdtrain-F1_score:0.554327\tdval-F1_score:0.330665\n",
      "[111]\tdtrain-F1_score:0.556983\tdval-F1_score:0.32944\n",
      "[112]\tdtrain-F1_score:0.56214\tdval-F1_score:0.331438\n",
      "[113]\tdtrain-F1_score:0.563357\tdval-F1_score:0.331977\n",
      "[114]\tdtrain-F1_score:0.565105\tdval-F1_score:0.331153\n",
      "[115]\tdtrain-F1_score:0.566394\tdval-F1_score:0.331695\n",
      "[116]\tdtrain-F1_score:0.568161\tdval-F1_score:0.332924\n",
      "[117]\tdtrain-F1_score:0.573889\tdval-F1_score:0.333609\n",
      "[118]\tdtrain-F1_score:0.575946\tdval-F1_score:0.337329\n",
      "[119]\tdtrain-F1_score:0.577529\tdval-F1_score:0.33873\n",
      "[120]\tdtrain-F1_score:0.579332\tdval-F1_score:0.339025\n",
      "[121]\tdtrain-F1_score:0.581489\tdval-F1_score:0.340604\n",
      "[122]\tdtrain-F1_score:0.58385\tdval-F1_score:0.341053\n",
      "[123]\tdtrain-F1_score:0.589009\tdval-F1_score:0.341691\n",
      "[124]\tdtrain-F1_score:0.590472\tdval-F1_score:0.341443\n",
      "[125]\tdtrain-F1_score:0.593485\tdval-F1_score:0.339332\n",
      "[126]\tdtrain-F1_score:0.593614\tdval-F1_score:0.339477\n",
      "[127]\tdtrain-F1_score:0.594454\tdval-F1_score:0.338356\n",
      "[128]\tdtrain-F1_score:0.595555\tdval-F1_score:0.340645\n",
      "[129]\tdtrain-F1_score:0.59666\tdval-F1_score:0.341232\n",
      "[130]\tdtrain-F1_score:0.598423\tdval-F1_score:0.341843\n",
      "[131]\tdtrain-F1_score:0.600527\tdval-F1_score:0.340702\n",
      "[132]\tdtrain-F1_score:0.601982\tdval-F1_score:0.343601\n",
      "[133]\tdtrain-F1_score:0.603711\tdval-F1_score:0.343154\n",
      "[134]\tdtrain-F1_score:0.606121\tdval-F1_score:0.341421\n",
      "[135]\tdtrain-F1_score:0.608077\tdval-F1_score:0.341591\n",
      "[136]\tdtrain-F1_score:0.609772\tdval-F1_score:0.34076\n",
      "[137]\tdtrain-F1_score:0.611204\tdval-F1_score:0.342059\n",
      "[138]\tdtrain-F1_score:0.612643\tdval-F1_score:0.345009\n",
      "[139]\tdtrain-F1_score:0.614364\tdval-F1_score:0.34507\n",
      "[140]\tdtrain-F1_score:0.617069\tdval-F1_score:0.345286\n",
      "[141]\tdtrain-F1_score:0.618256\tdval-F1_score:0.343904\n",
      "[142]\tdtrain-F1_score:0.619377\tdval-F1_score:0.344519\n",
      "[143]\tdtrain-F1_score:0.621914\tdval-F1_score:0.344982\n",
      "[144]\tdtrain-F1_score:0.624757\tdval-F1_score:0.347003\n",
      "[145]\tdtrain-F1_score:0.626547\tdval-F1_score:0.347669\n",
      "[146]\tdtrain-F1_score:0.627627\tdval-F1_score:0.348499\n",
      "[147]\tdtrain-F1_score:0.629941\tdval-F1_score:0.346837\n",
      "[148]\tdtrain-F1_score:0.631615\tdval-F1_score:0.349886\n",
      "[149]\tdtrain-F1_score:0.632346\tdval-F1_score:0.35\n",
      "[150]\tdtrain-F1_score:0.633372\tdval-F1_score:0.350205\n",
      "[151]\tdtrain-F1_score:0.636099\tdval-F1_score:0.350298\n",
      "[152]\tdtrain-F1_score:0.641926\tdval-F1_score:0.350715\n",
      "[153]\tdtrain-F1_score:0.645184\tdval-F1_score:0.351414\n",
      "[154]\tdtrain-F1_score:0.646405\tdval-F1_score:0.350324\n",
      "[155]\tdtrain-F1_score:0.648168\tdval-F1_score:0.353051\n",
      "[156]\tdtrain-F1_score:0.649169\tdval-F1_score:0.354312\n",
      "[157]\tdtrain-F1_score:0.650947\tdval-F1_score:0.354914\n",
      "[158]\tdtrain-F1_score:0.652501\tdval-F1_score:0.35514\n",
      "[159]\tdtrain-F1_score:0.654141\tdval-F1_score:0.354763\n",
      "[160]\tdtrain-F1_score:0.655553\tdval-F1_score:0.355702\n",
      "[161]\tdtrain-F1_score:0.658792\tdval-F1_score:0.35461\n",
      "[162]\tdtrain-F1_score:0.660464\tdval-F1_score:0.353941\n",
      "[163]\tdtrain-F1_score:0.661183\tdval-F1_score:0.353053\n",
      "[164]\tdtrain-F1_score:0.662626\tdval-F1_score:0.353664\n",
      "[165]\tdtrain-F1_score:0.664559\tdval-F1_score:0.352159\n",
      "[166]\tdtrain-F1_score:0.666179\tdval-F1_score:0.351931\n",
      "[167]\tdtrain-F1_score:0.666017\tdval-F1_score:0.351931\n",
      "[168]\tdtrain-F1_score:0.667644\tdval-F1_score:0.35249\n",
      "[169]\tdtrain-F1_score:0.668869\tdval-F1_score:0.352153\n",
      "[170]\tdtrain-F1_score:0.670264\tdval-F1_score:0.353618\n",
      "Stopping. Best iteration:\n",
      "[160]\tdtrain-F1_score:0.655553\tdval-F1_score:0.355702\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.188333\tdval-F1_score:0.177471\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.199504\tdval-F1_score:0.186568\n",
      "[2]\tdtrain-F1_score:0.210004\tdval-F1_score:0.195818\n",
      "[3]\tdtrain-F1_score:0.216116\tdval-F1_score:0.197286\n",
      "[4]\tdtrain-F1_score:0.218308\tdval-F1_score:0.198624\n",
      "[5]\tdtrain-F1_score:0.222539\tdval-F1_score:0.201429\n",
      "[6]\tdtrain-F1_score:0.227079\tdval-F1_score:0.202958\n",
      "[7]\tdtrain-F1_score:0.232686\tdval-F1_score:0.206738\n",
      "[8]\tdtrain-F1_score:0.237953\tdval-F1_score:0.211117\n",
      "[9]\tdtrain-F1_score:0.242922\tdval-F1_score:0.212868\n",
      "[10]\tdtrain-F1_score:0.25007\tdval-F1_score:0.217795\n",
      "[11]\tdtrain-F1_score:0.251385\tdval-F1_score:0.218112\n",
      "[12]\tdtrain-F1_score:0.258018\tdval-F1_score:0.223003\n",
      "[13]\tdtrain-F1_score:0.262002\tdval-F1_score:0.226624\n",
      "[14]\tdtrain-F1_score:0.264712\tdval-F1_score:0.227282\n",
      "[15]\tdtrain-F1_score:0.269323\tdval-F1_score:0.23055\n",
      "[16]\tdtrain-F1_score:0.274018\tdval-F1_score:0.23483\n",
      "[17]\tdtrain-F1_score:0.276901\tdval-F1_score:0.234434\n",
      "[18]\tdtrain-F1_score:0.281561\tdval-F1_score:0.238045\n",
      "[19]\tdtrain-F1_score:0.285609\tdval-F1_score:0.239931\n",
      "[20]\tdtrain-F1_score:0.288229\tdval-F1_score:0.242319\n",
      "[21]\tdtrain-F1_score:0.293836\tdval-F1_score:0.245279\n",
      "[22]\tdtrain-F1_score:0.296613\tdval-F1_score:0.245787\n",
      "[23]\tdtrain-F1_score:0.298962\tdval-F1_score:0.248251\n",
      "[24]\tdtrain-F1_score:0.301984\tdval-F1_score:0.248919\n",
      "[25]\tdtrain-F1_score:0.308208\tdval-F1_score:0.250405\n",
      "[26]\tdtrain-F1_score:0.309624\tdval-F1_score:0.251217\n",
      "[27]\tdtrain-F1_score:0.314143\tdval-F1_score:0.252998\n",
      "[28]\tdtrain-F1_score:0.321125\tdval-F1_score:0.255881\n",
      "[29]\tdtrain-F1_score:0.325796\tdval-F1_score:0.25952\n",
      "[30]\tdtrain-F1_score:0.329232\tdval-F1_score:0.261083\n",
      "[31]\tdtrain-F1_score:0.331928\tdval-F1_score:0.261192\n",
      "[32]\tdtrain-F1_score:0.335308\tdval-F1_score:0.262328\n",
      "[33]\tdtrain-F1_score:0.339051\tdval-F1_score:0.264313\n",
      "[34]\tdtrain-F1_score:0.340258\tdval-F1_score:0.264266\n",
      "[35]\tdtrain-F1_score:0.341387\tdval-F1_score:0.26514\n",
      "[36]\tdtrain-F1_score:0.34548\tdval-F1_score:0.268664\n",
      "[37]\tdtrain-F1_score:0.34826\tdval-F1_score:0.272017\n",
      "[38]\tdtrain-F1_score:0.350728\tdval-F1_score:0.273563\n",
      "[39]\tdtrain-F1_score:0.35366\tdval-F1_score:0.275278\n",
      "[40]\tdtrain-F1_score:0.355347\tdval-F1_score:0.276155\n",
      "[41]\tdtrain-F1_score:0.361355\tdval-F1_score:0.276624\n",
      "[42]\tdtrain-F1_score:0.363066\tdval-F1_score:0.276291\n",
      "[43]\tdtrain-F1_score:0.365615\tdval-F1_score:0.276555\n",
      "[44]\tdtrain-F1_score:0.366712\tdval-F1_score:0.275824\n",
      "[45]\tdtrain-F1_score:0.371284\tdval-F1_score:0.278544\n",
      "[46]\tdtrain-F1_score:0.375819\tdval-F1_score:0.280457\n",
      "[47]\tdtrain-F1_score:0.378453\tdval-F1_score:0.281066\n",
      "[48]\tdtrain-F1_score:0.382301\tdval-F1_score:0.280057\n",
      "[49]\tdtrain-F1_score:0.38642\tdval-F1_score:0.282353\n",
      "[50]\tdtrain-F1_score:0.389958\tdval-F1_score:0.284714\n",
      "[51]\tdtrain-F1_score:0.39192\tdval-F1_score:0.283705\n",
      "[52]\tdtrain-F1_score:0.395992\tdval-F1_score:0.284786\n",
      "[53]\tdtrain-F1_score:0.399736\tdval-F1_score:0.285714\n",
      "[54]\tdtrain-F1_score:0.401354\tdval-F1_score:0.286486\n",
      "[55]\tdtrain-F1_score:0.407076\tdval-F1_score:0.288754\n",
      "[56]\tdtrain-F1_score:0.411312\tdval-F1_score:0.289401\n",
      "[57]\tdtrain-F1_score:0.415\tdval-F1_score:0.289433\n",
      "[58]\tdtrain-F1_score:0.416711\tdval-F1_score:0.288999\n",
      "[59]\tdtrain-F1_score:0.420274\tdval-F1_score:0.291171\n",
      "[60]\tdtrain-F1_score:0.421443\tdval-F1_score:0.291916\n",
      "[61]\tdtrain-F1_score:0.424021\tdval-F1_score:0.290231\n",
      "[62]\tdtrain-F1_score:0.427542\tdval-F1_score:0.293367\n",
      "[63]\tdtrain-F1_score:0.431122\tdval-F1_score:0.294913\n",
      "[64]\tdtrain-F1_score:0.433691\tdval-F1_score:0.296296\n",
      "[65]\tdtrain-F1_score:0.435282\tdval-F1_score:0.296417\n",
      "[66]\tdtrain-F1_score:0.438499\tdval-F1_score:0.295835\n",
      "[67]\tdtrain-F1_score:0.439559\tdval-F1_score:0.295835\n",
      "[68]\tdtrain-F1_score:0.441336\tdval-F1_score:0.296467\n",
      "[69]\tdtrain-F1_score:0.442051\tdval-F1_score:0.295191\n",
      "[70]\tdtrain-F1_score:0.444897\tdval-F1_score:0.297532\n",
      "[71]\tdtrain-F1_score:0.450309\tdval-F1_score:0.298246\n",
      "[72]\tdtrain-F1_score:0.451202\tdval-F1_score:0.298477\n",
      "[73]\tdtrain-F1_score:0.45576\tdval-F1_score:0.300951\n",
      "[74]\tdtrain-F1_score:0.460967\tdval-F1_score:0.301511\n",
      "[75]\tdtrain-F1_score:0.462881\tdval-F1_score:0.302632\n",
      "[76]\tdtrain-F1_score:0.466268\tdval-F1_score:0.307692\n",
      "[77]\tdtrain-F1_score:0.469516\tdval-F1_score:0.308995\n",
      "[78]\tdtrain-F1_score:0.471177\tdval-F1_score:0.309431\n",
      "[79]\tdtrain-F1_score:0.475486\tdval-F1_score:0.309709\n",
      "[80]\tdtrain-F1_score:0.478653\tdval-F1_score:0.311651\n",
      "[81]\tdtrain-F1_score:0.481998\tdval-F1_score:0.313895\n",
      "[82]\tdtrain-F1_score:0.483363\tdval-F1_score:0.313357\n",
      "[83]\tdtrain-F1_score:0.485684\tdval-F1_score:0.315942\n",
      "[84]\tdtrain-F1_score:0.487766\tdval-F1_score:0.318165\n",
      "[85]\tdtrain-F1_score:0.492427\tdval-F1_score:0.318798\n",
      "[86]\tdtrain-F1_score:0.494793\tdval-F1_score:0.319352\n",
      "[87]\tdtrain-F1_score:0.499223\tdval-F1_score:0.321362\n",
      "[88]\tdtrain-F1_score:0.501607\tdval-F1_score:0.322413\n",
      "[89]\tdtrain-F1_score:0.504154\tdval-F1_score:0.321015\n",
      "[90]\tdtrain-F1_score:0.508658\tdval-F1_score:0.320932\n",
      "[91]\tdtrain-F1_score:0.509606\tdval-F1_score:0.31991\n",
      "[92]\tdtrain-F1_score:0.511563\tdval-F1_score:0.322826\n",
      "[93]\tdtrain-F1_score:0.516641\tdval-F1_score:0.31945\n",
      "[94]\tdtrain-F1_score:0.518308\tdval-F1_score:0.31874\n",
      "[95]\tdtrain-F1_score:0.5266\tdval-F1_score:0.319406\n",
      "[96]\tdtrain-F1_score:0.529457\tdval-F1_score:0.320843\n",
      "[97]\tdtrain-F1_score:0.532761\tdval-F1_score:0.318466\n",
      "[98]\tdtrain-F1_score:0.534377\tdval-F1_score:0.319937\n",
      "[99]\tdtrain-F1_score:0.536528\tdval-F1_score:0.319873\n",
      "[100]\tdtrain-F1_score:0.538644\tdval-F1_score:0.318218\n",
      "[101]\tdtrain-F1_score:0.54099\tdval-F1_score:0.319456\n",
      "[102]\tdtrain-F1_score:0.545201\tdval-F1_score:0.31984\n",
      "Stopping. Best iteration:\n",
      "[92]\tdtrain-F1_score:0.511563\tdval-F1_score:0.322826\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.177686\tdval-F1_score:0.163757\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.199238\tdval-F1_score:0.184434\n",
      "[2]\tdtrain-F1_score:0.208453\tdval-F1_score:0.188619\n",
      "[3]\tdtrain-F1_score:0.222643\tdval-F1_score:0.197641\n",
      "[4]\tdtrain-F1_score:0.225017\tdval-F1_score:0.201036\n",
      "[5]\tdtrain-F1_score:0.232165\tdval-F1_score:0.20636\n",
      "[6]\tdtrain-F1_score:0.239013\tdval-F1_score:0.210564\n",
      "[7]\tdtrain-F1_score:0.242874\tdval-F1_score:0.212672\n",
      "[8]\tdtrain-F1_score:0.244876\tdval-F1_score:0.212412\n",
      "[9]\tdtrain-F1_score:0.249144\tdval-F1_score:0.215367\n",
      "[10]\tdtrain-F1_score:0.25561\tdval-F1_score:0.218684\n",
      "[11]\tdtrain-F1_score:0.257372\tdval-F1_score:0.218224\n",
      "[12]\tdtrain-F1_score:0.264382\tdval-F1_score:0.224728\n",
      "[13]\tdtrain-F1_score:0.268186\tdval-F1_score:0.226045\n",
      "[14]\tdtrain-F1_score:0.268593\tdval-F1_score:0.226324\n",
      "[15]\tdtrain-F1_score:0.273795\tdval-F1_score:0.230154\n",
      "[16]\tdtrain-F1_score:0.279618\tdval-F1_score:0.234773\n",
      "[17]\tdtrain-F1_score:0.283961\tdval-F1_score:0.236803\n",
      "[18]\tdtrain-F1_score:0.288132\tdval-F1_score:0.239861\n",
      "[19]\tdtrain-F1_score:0.293904\tdval-F1_score:0.242117\n",
      "[20]\tdtrain-F1_score:0.296503\tdval-F1_score:0.242816\n",
      "[21]\tdtrain-F1_score:0.299048\tdval-F1_score:0.244824\n",
      "[22]\tdtrain-F1_score:0.303542\tdval-F1_score:0.247099\n",
      "[23]\tdtrain-F1_score:0.307579\tdval-F1_score:0.245823\n",
      "[24]\tdtrain-F1_score:0.310575\tdval-F1_score:0.247919\n",
      "[25]\tdtrain-F1_score:0.315356\tdval-F1_score:0.250118\n",
      "[26]\tdtrain-F1_score:0.318333\tdval-F1_score:0.250654\n",
      "[27]\tdtrain-F1_score:0.321874\tdval-F1_score:0.254136\n",
      "[28]\tdtrain-F1_score:0.323486\tdval-F1_score:0.254642\n",
      "[29]\tdtrain-F1_score:0.327067\tdval-F1_score:0.256148\n",
      "[30]\tdtrain-F1_score:0.330061\tdval-F1_score:0.256687\n",
      "[31]\tdtrain-F1_score:0.332073\tdval-F1_score:0.257044\n",
      "[32]\tdtrain-F1_score:0.34003\tdval-F1_score:0.259642\n",
      "[33]\tdtrain-F1_score:0.341993\tdval-F1_score:0.261663\n",
      "[34]\tdtrain-F1_score:0.343069\tdval-F1_score:0.261777\n",
      "[35]\tdtrain-F1_score:0.34836\tdval-F1_score:0.261677\n",
      "[36]\tdtrain-F1_score:0.35097\tdval-F1_score:0.2642\n",
      "[37]\tdtrain-F1_score:0.352728\tdval-F1_score:0.264852\n",
      "[38]\tdtrain-F1_score:0.356181\tdval-F1_score:0.266596\n",
      "[39]\tdtrain-F1_score:0.360641\tdval-F1_score:0.269138\n",
      "[40]\tdtrain-F1_score:0.363673\tdval-F1_score:0.269803\n",
      "[41]\tdtrain-F1_score:0.365625\tdval-F1_score:0.269596\n",
      "[42]\tdtrain-F1_score:0.369953\tdval-F1_score:0.267764\n",
      "[43]\tdtrain-F1_score:0.372207\tdval-F1_score:0.271448\n",
      "[44]\tdtrain-F1_score:0.373945\tdval-F1_score:0.27303\n",
      "[45]\tdtrain-F1_score:0.377137\tdval-F1_score:0.272473\n",
      "[46]\tdtrain-F1_score:0.382322\tdval-F1_score:0.273625\n",
      "[47]\tdtrain-F1_score:0.385023\tdval-F1_score:0.275803\n",
      "[48]\tdtrain-F1_score:0.386393\tdval-F1_score:0.276196\n",
      "[49]\tdtrain-F1_score:0.389349\tdval-F1_score:0.277108\n",
      "[50]\tdtrain-F1_score:0.393762\tdval-F1_score:0.27741\n",
      "[51]\tdtrain-F1_score:0.395447\tdval-F1_score:0.276301\n",
      "[52]\tdtrain-F1_score:0.398248\tdval-F1_score:0.278474\n",
      "[53]\tdtrain-F1_score:0.400882\tdval-F1_score:0.280866\n",
      "[54]\tdtrain-F1_score:0.403192\tdval-F1_score:0.280165\n",
      "[55]\tdtrain-F1_score:0.407162\tdval-F1_score:0.281046\n",
      "[56]\tdtrain-F1_score:0.412262\tdval-F1_score:0.281175\n",
      "[57]\tdtrain-F1_score:0.416349\tdval-F1_score:0.280182\n",
      "[58]\tdtrain-F1_score:0.41832\tdval-F1_score:0.283537\n",
      "[59]\tdtrain-F1_score:0.419637\tdval-F1_score:0.283883\n",
      "[60]\tdtrain-F1_score:0.423345\tdval-F1_score:0.286508\n",
      "[61]\tdtrain-F1_score:0.426875\tdval-F1_score:0.286606\n",
      "[62]\tdtrain-F1_score:0.430202\tdval-F1_score:0.289564\n",
      "[63]\tdtrain-F1_score:0.43585\tdval-F1_score:0.290446\n",
      "[64]\tdtrain-F1_score:0.438789\tdval-F1_score:0.291293\n",
      "[65]\tdtrain-F1_score:0.442197\tdval-F1_score:0.291909\n",
      "[66]\tdtrain-F1_score:0.443885\tdval-F1_score:0.292952\n",
      "[67]\tdtrain-F1_score:0.44807\tdval-F1_score:0.297803\n",
      "[68]\tdtrain-F1_score:0.453309\tdval-F1_score:0.299934\n",
      "[69]\tdtrain-F1_score:0.454818\tdval-F1_score:0.299767\n",
      "[70]\tdtrain-F1_score:0.456184\tdval-F1_score:0.300767\n",
      "[71]\tdtrain-F1_score:0.459907\tdval-F1_score:0.301646\n",
      "[72]\tdtrain-F1_score:0.462163\tdval-F1_score:0.303481\n",
      "[73]\tdtrain-F1_score:0.466411\tdval-F1_score:0.302041\n",
      "[74]\tdtrain-F1_score:0.472256\tdval-F1_score:0.302069\n",
      "[75]\tdtrain-F1_score:0.474635\tdval-F1_score:0.30571\n",
      "[76]\tdtrain-F1_score:0.47733\tdval-F1_score:0.305808\n",
      "[77]\tdtrain-F1_score:0.482813\tdval-F1_score:0.309549\n",
      "[78]\tdtrain-F1_score:0.486294\tdval-F1_score:0.307747\n",
      "[79]\tdtrain-F1_score:0.488254\tdval-F1_score:0.308026\n",
      "[80]\tdtrain-F1_score:0.492743\tdval-F1_score:0.309541\n",
      "[81]\tdtrain-F1_score:0.495468\tdval-F1_score:0.311565\n",
      "[82]\tdtrain-F1_score:0.499178\tdval-F1_score:0.311241\n",
      "[83]\tdtrain-F1_score:0.502528\tdval-F1_score:0.310408\n",
      "[84]\tdtrain-F1_score:0.503825\tdval-F1_score:0.310801\n",
      "[85]\tdtrain-F1_score:0.506533\tdval-F1_score:0.310536\n",
      "[86]\tdtrain-F1_score:0.507474\tdval-F1_score:0.311045\n",
      "[87]\tdtrain-F1_score:0.511319\tdval-F1_score:0.309985\n",
      "[88]\tdtrain-F1_score:0.512277\tdval-F1_score:0.310861\n",
      "[89]\tdtrain-F1_score:0.514932\tdval-F1_score:0.310709\n",
      "[90]\tdtrain-F1_score:0.520571\tdval-F1_score:0.310201\n",
      "[91]\tdtrain-F1_score:0.521714\tdval-F1_score:0.309795\n",
      "Stopping. Best iteration:\n",
      "[81]\tdtrain-F1_score:0.495468\tdval-F1_score:0.311565\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.207346\tdval-F1_score:0.182696\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.21705\tdval-F1_score:0.190837\n",
      "[2]\tdtrain-F1_score:0.218327\tdval-F1_score:0.190686\n",
      "[3]\tdtrain-F1_score:0.227499\tdval-F1_score:0.198023\n",
      "[4]\tdtrain-F1_score:0.2325\tdval-F1_score:0.201871\n",
      "[5]\tdtrain-F1_score:0.232854\tdval-F1_score:0.202006\n",
      "[6]\tdtrain-F1_score:0.242185\tdval-F1_score:0.206935\n",
      "[7]\tdtrain-F1_score:0.244372\tdval-F1_score:0.206042\n",
      "[8]\tdtrain-F1_score:0.24887\tdval-F1_score:0.209804\n",
      "[9]\tdtrain-F1_score:0.251333\tdval-F1_score:0.212961\n",
      "[10]\tdtrain-F1_score:0.256235\tdval-F1_score:0.214466\n",
      "[11]\tdtrain-F1_score:0.258991\tdval-F1_score:0.216778\n",
      "[12]\tdtrain-F1_score:0.268372\tdval-F1_score:0.222132\n",
      "[13]\tdtrain-F1_score:0.271346\tdval-F1_score:0.224486\n",
      "[14]\tdtrain-F1_score:0.275773\tdval-F1_score:0.226548\n",
      "[15]\tdtrain-F1_score:0.280586\tdval-F1_score:0.231419\n",
      "[16]\tdtrain-F1_score:0.284026\tdval-F1_score:0.232568\n",
      "[17]\tdtrain-F1_score:0.288838\tdval-F1_score:0.233434\n",
      "[18]\tdtrain-F1_score:0.29249\tdval-F1_score:0.233166\n",
      "[19]\tdtrain-F1_score:0.294551\tdval-F1_score:0.234543\n",
      "[20]\tdtrain-F1_score:0.299983\tdval-F1_score:0.235635\n",
      "[21]\tdtrain-F1_score:0.302817\tdval-F1_score:0.239602\n",
      "[22]\tdtrain-F1_score:0.305858\tdval-F1_score:0.240218\n",
      "[23]\tdtrain-F1_score:0.309212\tdval-F1_score:0.241324\n",
      "[24]\tdtrain-F1_score:0.311436\tdval-F1_score:0.243012\n",
      "[25]\tdtrain-F1_score:0.318019\tdval-F1_score:0.244361\n",
      "[26]\tdtrain-F1_score:0.326951\tdval-F1_score:0.251271\n",
      "[27]\tdtrain-F1_score:0.331341\tdval-F1_score:0.250798\n",
      "[28]\tdtrain-F1_score:0.335308\tdval-F1_score:0.252845\n",
      "[29]\tdtrain-F1_score:0.336885\tdval-F1_score:0.252043\n",
      "[30]\tdtrain-F1_score:0.338774\tdval-F1_score:0.252239\n",
      "[31]\tdtrain-F1_score:0.343942\tdval-F1_score:0.253018\n",
      "[32]\tdtrain-F1_score:0.345846\tdval-F1_score:0.255491\n",
      "[33]\tdtrain-F1_score:0.350274\tdval-F1_score:0.259401\n",
      "[34]\tdtrain-F1_score:0.352293\tdval-F1_score:0.258763\n",
      "[35]\tdtrain-F1_score:0.355518\tdval-F1_score:0.258282\n",
      "[36]\tdtrain-F1_score:0.359662\tdval-F1_score:0.257659\n",
      "[37]\tdtrain-F1_score:0.361152\tdval-F1_score:0.260343\n",
      "[38]\tdtrain-F1_score:0.366956\tdval-F1_score:0.263383\n",
      "[39]\tdtrain-F1_score:0.369697\tdval-F1_score:0.263713\n",
      "[40]\tdtrain-F1_score:0.371386\tdval-F1_score:0.263885\n",
      "[41]\tdtrain-F1_score:0.374862\tdval-F1_score:0.263574\n",
      "[42]\tdtrain-F1_score:0.377722\tdval-F1_score:0.265014\n",
      "[43]\tdtrain-F1_score:0.38052\tdval-F1_score:0.265295\n",
      "[44]\tdtrain-F1_score:0.384182\tdval-F1_score:0.264556\n",
      "[45]\tdtrain-F1_score:0.386936\tdval-F1_score:0.264463\n",
      "[46]\tdtrain-F1_score:0.388038\tdval-F1_score:0.264991\n",
      "[47]\tdtrain-F1_score:0.390205\tdval-F1_score:0.266475\n",
      "[48]\tdtrain-F1_score:0.398335\tdval-F1_score:0.268464\n",
      "[49]\tdtrain-F1_score:0.401176\tdval-F1_score:0.270686\n",
      "[50]\tdtrain-F1_score:0.406709\tdval-F1_score:0.274416\n",
      "[51]\tdtrain-F1_score:0.409394\tdval-F1_score:0.274924\n",
      "[52]\tdtrain-F1_score:0.412521\tdval-F1_score:0.276156\n",
      "[53]\tdtrain-F1_score:0.416431\tdval-F1_score:0.277846\n",
      "[54]\tdtrain-F1_score:0.418348\tdval-F1_score:0.281577\n",
      "[55]\tdtrain-F1_score:0.424601\tdval-F1_score:0.281269\n",
      "[56]\tdtrain-F1_score:0.425562\tdval-F1_score:0.281445\n",
      "[57]\tdtrain-F1_score:0.428403\tdval-F1_score:0.28338\n",
      "[58]\tdtrain-F1_score:0.43257\tdval-F1_score:0.284264\n",
      "[59]\tdtrain-F1_score:0.434187\tdval-F1_score:0.2839\n",
      "[60]\tdtrain-F1_score:0.434845\tdval-F1_score:0.281807\n",
      "[61]\tdtrain-F1_score:0.437215\tdval-F1_score:0.28133\n",
      "[62]\tdtrain-F1_score:0.442804\tdval-F1_score:0.283771\n",
      "[63]\tdtrain-F1_score:0.448239\tdval-F1_score:0.286276\n",
      "[64]\tdtrain-F1_score:0.450607\tdval-F1_score:0.286091\n",
      "[65]\tdtrain-F1_score:0.452474\tdval-F1_score:0.287037\n",
      "[66]\tdtrain-F1_score:0.457531\tdval-F1_score:0.289245\n",
      "[67]\tdtrain-F1_score:0.460074\tdval-F1_score:0.290995\n",
      "[68]\tdtrain-F1_score:0.462972\tdval-F1_score:0.290529\n",
      "[69]\tdtrain-F1_score:0.464034\tdval-F1_score:0.29237\n",
      "[70]\tdtrain-F1_score:0.466519\tdval-F1_score:0.292203\n",
      "[71]\tdtrain-F1_score:0.469496\tdval-F1_score:0.289033\n",
      "[72]\tdtrain-F1_score:0.47135\tdval-F1_score:0.290822\n",
      "[73]\tdtrain-F1_score:0.475701\tdval-F1_score:0.289765\n",
      "[74]\tdtrain-F1_score:0.478158\tdval-F1_score:0.288881\n",
      "[75]\tdtrain-F1_score:0.48026\tdval-F1_score:0.289703\n",
      "[76]\tdtrain-F1_score:0.482338\tdval-F1_score:0.289216\n",
      "[77]\tdtrain-F1_score:0.484994\tdval-F1_score:0.290424\n",
      "[78]\tdtrain-F1_score:0.485468\tdval-F1_score:0.290028\n",
      "[79]\tdtrain-F1_score:0.490397\tdval-F1_score:0.291726\n",
      "Stopping. Best iteration:\n",
      "[69]\tdtrain-F1_score:0.464034\tdval-F1_score:0.29237\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tdtrain-F1_score:0.180742\tdval-F1_score:0.163859\n",
      "Multiple eval metrics have been passed: 'dval-F1_score' will be used for early stopping.\n",
      "\n",
      "Will train until dval-F1_score hasn't improved in 10 rounds.\n",
      "[1]\tdtrain-F1_score:0.204446\tdval-F1_score:0.183066\n",
      "[2]\tdtrain-F1_score:0.211604\tdval-F1_score:0.18507\n",
      "[3]\tdtrain-F1_score:0.217064\tdval-F1_score:0.190097\n",
      "[4]\tdtrain-F1_score:0.222204\tdval-F1_score:0.192127\n",
      "[5]\tdtrain-F1_score:0.229536\tdval-F1_score:0.197821\n",
      "[6]\tdtrain-F1_score:0.231997\tdval-F1_score:0.200405\n",
      "[7]\tdtrain-F1_score:0.239714\tdval-F1_score:0.206752\n",
      "[8]\tdtrain-F1_score:0.245945\tdval-F1_score:0.209623\n",
      "[9]\tdtrain-F1_score:0.247903\tdval-F1_score:0.211837\n",
      "[10]\tdtrain-F1_score:0.255474\tdval-F1_score:0.215209\n",
      "[11]\tdtrain-F1_score:0.260382\tdval-F1_score:0.217725\n",
      "[12]\tdtrain-F1_score:0.261667\tdval-F1_score:0.218803\n",
      "[13]\tdtrain-F1_score:0.267333\tdval-F1_score:0.2212\n",
      "[14]\tdtrain-F1_score:0.27201\tdval-F1_score:0.225385\n",
      "[15]\tdtrain-F1_score:0.274843\tdval-F1_score:0.226475\n",
      "[16]\tdtrain-F1_score:0.277396\tdval-F1_score:0.228196\n",
      "[17]\tdtrain-F1_score:0.280564\tdval-F1_score:0.229251\n",
      "[18]\tdtrain-F1_score:0.285188\tdval-F1_score:0.2316\n",
      "[19]\tdtrain-F1_score:0.290843\tdval-F1_score:0.233945\n",
      "[20]\tdtrain-F1_score:0.296446\tdval-F1_score:0.238024\n",
      "[21]\tdtrain-F1_score:0.298767\tdval-F1_score:0.239431\n",
      "[22]\tdtrain-F1_score:0.302214\tdval-F1_score:0.241237\n",
      "[23]\tdtrain-F1_score:0.307068\tdval-F1_score:0.243935\n",
      "[24]\tdtrain-F1_score:0.309872\tdval-F1_score:0.246044\n",
      "[25]\tdtrain-F1_score:0.313493\tdval-F1_score:0.247478\n",
      "[26]\tdtrain-F1_score:0.31582\tdval-F1_score:0.248871\n",
      "[27]\tdtrain-F1_score:0.318615\tdval-F1_score:0.248235\n",
      "[28]\tdtrain-F1_score:0.320678\tdval-F1_score:0.247711\n",
      "[29]\tdtrain-F1_score:0.323743\tdval-F1_score:0.250928\n",
      "[30]\tdtrain-F1_score:0.328866\tdval-F1_score:0.251994\n",
      "[31]\tdtrain-F1_score:0.332114\tdval-F1_score:0.255875\n",
      "[32]\tdtrain-F1_score:0.333293\tdval-F1_score:0.255935\n",
      "[33]\tdtrain-F1_score:0.334993\tdval-F1_score:0.257095\n",
      "[34]\tdtrain-F1_score:0.338867\tdval-F1_score:0.25934\n",
      "[35]\tdtrain-F1_score:0.344719\tdval-F1_score:0.26002\n",
      "[36]\tdtrain-F1_score:0.348237\tdval-F1_score:0.26087\n",
      "[37]\tdtrain-F1_score:0.351085\tdval-F1_score:0.262897\n",
      "[38]\tdtrain-F1_score:0.353292\tdval-F1_score:0.26417\n",
      "[39]\tdtrain-F1_score:0.357625\tdval-F1_score:0.268046\n",
      "[40]\tdtrain-F1_score:0.360463\tdval-F1_score:0.269995\n",
      "[41]\tdtrain-F1_score:0.363709\tdval-F1_score:0.271561\n",
      "[42]\tdtrain-F1_score:0.365256\tdval-F1_score:0.270994\n",
      "[43]\tdtrain-F1_score:0.369308\tdval-F1_score:0.273673\n",
      "[44]\tdtrain-F1_score:0.372738\tdval-F1_score:0.275771\n",
      "[45]\tdtrain-F1_score:0.37494\tdval-F1_score:0.276264\n",
      "[46]\tdtrain-F1_score:0.380461\tdval-F1_score:0.277837\n",
      "[47]\tdtrain-F1_score:0.381958\tdval-F1_score:0.27942\n",
      "[48]\tdtrain-F1_score:0.383083\tdval-F1_score:0.279408\n",
      "[49]\tdtrain-F1_score:0.385163\tdval-F1_score:0.278529\n",
      "[50]\tdtrain-F1_score:0.390422\tdval-F1_score:0.282023\n",
      "[51]\tdtrain-F1_score:0.391878\tdval-F1_score:0.281527\n",
      "[52]\tdtrain-F1_score:0.394084\tdval-F1_score:0.280311\n",
      "[53]\tdtrain-F1_score:0.398628\tdval-F1_score:0.28338\n",
      "[54]\tdtrain-F1_score:0.401824\tdval-F1_score:0.283531\n",
      "[55]\tdtrain-F1_score:0.404563\tdval-F1_score:0.285146\n",
      "[56]\tdtrain-F1_score:0.406611\tdval-F1_score:0.284983\n",
      "[57]\tdtrain-F1_score:0.411947\tdval-F1_score:0.286371\n",
      "[58]\tdtrain-F1_score:0.415582\tdval-F1_score:0.289359\n",
      "[59]\tdtrain-F1_score:0.416438\tdval-F1_score:0.288876\n",
      "[60]\tdtrain-F1_score:0.421191\tdval-F1_score:0.291851\n",
      "[61]\tdtrain-F1_score:0.425125\tdval-F1_score:0.289864\n",
      "[62]\tdtrain-F1_score:0.426752\tdval-F1_score:0.288948\n",
      "[63]\tdtrain-F1_score:0.427587\tdval-F1_score:0.289482\n",
      "[64]\tdtrain-F1_score:0.431345\tdval-F1_score:0.29198\n",
      "[65]\tdtrain-F1_score:0.433397\tdval-F1_score:0.294242\n",
      "[66]\tdtrain-F1_score:0.436163\tdval-F1_score:0.296161\n",
      "[67]\tdtrain-F1_score:0.437385\tdval-F1_score:0.296975\n",
      "[68]\tdtrain-F1_score:0.439601\tdval-F1_score:0.297663\n",
      "[69]\tdtrain-F1_score:0.441947\tdval-F1_score:0.295931\n",
      "[70]\tdtrain-F1_score:0.442734\tdval-F1_score:0.295836\n",
      "[71]\tdtrain-F1_score:0.445332\tdval-F1_score:0.297526\n",
      "[72]\tdtrain-F1_score:0.446751\tdval-F1_score:0.298301\n",
      "[73]\tdtrain-F1_score:0.450655\tdval-F1_score:0.299397\n",
      "[74]\tdtrain-F1_score:0.451773\tdval-F1_score:0.300924\n",
      "[75]\tdtrain-F1_score:0.459019\tdval-F1_score:0.300322\n",
      "[76]\tdtrain-F1_score:0.461733\tdval-F1_score:0.299676\n",
      "[77]\tdtrain-F1_score:0.466223\tdval-F1_score:0.30104\n",
      "[78]\tdtrain-F1_score:0.469386\tdval-F1_score:0.30128\n",
      "[79]\tdtrain-F1_score:0.47141\tdval-F1_score:0.301514\n",
      "[80]\tdtrain-F1_score:0.475759\tdval-F1_score:0.305288\n",
      "[81]\tdtrain-F1_score:0.478592\tdval-F1_score:0.305379\n",
      "[82]\tdtrain-F1_score:0.480739\tdval-F1_score:0.307381\n",
      "[83]\tdtrain-F1_score:0.484875\tdval-F1_score:0.309048\n",
      "[84]\tdtrain-F1_score:0.49018\tdval-F1_score:0.309589\n",
      "[85]\tdtrain-F1_score:0.493499\tdval-F1_score:0.310131\n",
      "[86]\tdtrain-F1_score:0.496187\tdval-F1_score:0.309713\n",
      "[87]\tdtrain-F1_score:0.499087\tdval-F1_score:0.310477\n",
      "[88]\tdtrain-F1_score:0.502759\tdval-F1_score:0.313506\n",
      "[89]\tdtrain-F1_score:0.507945\tdval-F1_score:0.312786\n",
      "[90]\tdtrain-F1_score:0.51175\tdval-F1_score:0.311504\n",
      "[91]\tdtrain-F1_score:0.513047\tdval-F1_score:0.312389\n",
      "[92]\tdtrain-F1_score:0.514544\tdval-F1_score:0.314214\n",
      "[93]\tdtrain-F1_score:0.515077\tdval-F1_score:0.315113\n",
      "[94]\tdtrain-F1_score:0.516879\tdval-F1_score:0.315638\n",
      "[95]\tdtrain-F1_score:0.520076\tdval-F1_score:0.318067\n",
      "[96]\tdtrain-F1_score:0.525476\tdval-F1_score:0.320789\n",
      "[97]\tdtrain-F1_score:0.527504\tdval-F1_score:0.321376\n",
      "[98]\tdtrain-F1_score:0.529036\tdval-F1_score:0.321259\n",
      "[99]\tdtrain-F1_score:0.531041\tdval-F1_score:0.321704\n",
      "[100]\tdtrain-F1_score:0.534572\tdval-F1_score:0.322628\n",
      "[101]\tdtrain-F1_score:0.536461\tdval-F1_score:0.324084\n",
      "[102]\tdtrain-F1_score:0.539319\tdval-F1_score:0.324022\n",
      "[103]\tdtrain-F1_score:0.540439\tdval-F1_score:0.324143\n",
      "[104]\tdtrain-F1_score:0.543016\tdval-F1_score:0.325843\n",
      "[105]\tdtrain-F1_score:0.543881\tdval-F1_score:0.326087\n",
      "[106]\tdtrain-F1_score:0.545455\tdval-F1_score:0.324203\n",
      "[107]\tdtrain-F1_score:0.546709\tdval-F1_score:0.326439\n",
      "[108]\tdtrain-F1_score:0.549899\tdval-F1_score:0.324548\n",
      "[109]\tdtrain-F1_score:0.552289\tdval-F1_score:0.326284\n",
      "[110]\tdtrain-F1_score:0.556789\tdval-F1_score:0.326268\n",
      "[111]\tdtrain-F1_score:0.557812\tdval-F1_score:0.327135\n",
      "[112]\tdtrain-F1_score:0.558097\tdval-F1_score:0.327121\n",
      "[113]\tdtrain-F1_score:0.560156\tdval-F1_score:0.326702\n",
      "[114]\tdtrain-F1_score:0.562925\tdval-F1_score:0.327146\n",
      "[115]\tdtrain-F1_score:0.565253\tdval-F1_score:0.325834\n",
      "[116]\tdtrain-F1_score:0.568605\tdval-F1_score:0.326706\n",
      "[117]\tdtrain-F1_score:0.57422\tdval-F1_score:0.325069\n",
      "[118]\tdtrain-F1_score:0.577679\tdval-F1_score:0.326643\n",
      "[119]\tdtrain-F1_score:0.579332\tdval-F1_score:0.32687\n",
      "[120]\tdtrain-F1_score:0.579639\tdval-F1_score:0.326741\n",
      "[121]\tdtrain-F1_score:0.58007\tdval-F1_score:0.327791\n",
      "[122]\tdtrain-F1_score:0.582047\tdval-F1_score:0.33029\n",
      "[123]\tdtrain-F1_score:0.583787\tdval-F1_score:0.332006\n",
      "[124]\tdtrain-F1_score:0.586292\tdval-F1_score:0.330268\n",
      "[125]\tdtrain-F1_score:0.588755\tdval-F1_score:0.330253\n",
      "[126]\tdtrain-F1_score:0.589199\tdval-F1_score:0.330519\n",
      "[127]\tdtrain-F1_score:0.591111\tdval-F1_score:0.32983\n",
      "[128]\tdtrain-F1_score:0.595166\tdval-F1_score:0.331298\n",
      "[129]\tdtrain-F1_score:0.597508\tdval-F1_score:0.333879\n",
      "[130]\tdtrain-F1_score:0.600659\tdval-F1_score:0.3321\n",
      "[131]\tdtrain-F1_score:0.602314\tdval-F1_score:0.335126\n",
      "[132]\tdtrain-F1_score:0.603511\tdval-F1_score:0.335821\n",
      "[133]\tdtrain-F1_score:0.605249\tdval-F1_score:0.337219\n",
      "[134]\tdtrain-F1_score:0.610317\tdval-F1_score:0.341014\n",
      "[135]\tdtrain-F1_score:0.611273\tdval-F1_score:0.339908\n",
      "[136]\tdtrain-F1_score:0.612437\tdval-F1_score:0.341895\n",
      "[137]\tdtrain-F1_score:0.612506\tdval-F1_score:0.341607\n",
      "[138]\tdtrain-F1_score:0.619728\tdval-F1_score:0.342663\n",
      "[139]\tdtrain-F1_score:0.621985\tdval-F1_score:0.342809\n",
      "[140]\tdtrain-F1_score:0.62319\tdval-F1_score:0.34371\n",
      "[141]\tdtrain-F1_score:0.626979\tdval-F1_score:0.342442\n",
      "[142]\tdtrain-F1_score:0.62741\tdval-F1_score:0.346518\n",
      "[143]\tdtrain-F1_score:0.630668\tdval-F1_score:0.349481\n",
      "[144]\tdtrain-F1_score:0.630814\tdval-F1_score:0.350238\n",
      "[145]\tdtrain-F1_score:0.634843\tdval-F1_score:0.34828\n",
      "[146]\tdtrain-F1_score:0.63536\tdval-F1_score:0.348736\n",
      "[147]\tdtrain-F1_score:0.637137\tdval-F1_score:0.348551\n",
      "[148]\tdtrain-F1_score:0.638179\tdval-F1_score:0.347558\n",
      "[149]\tdtrain-F1_score:0.639448\tdval-F1_score:0.346867\n",
      "[150]\tdtrain-F1_score:0.6417\tdval-F1_score:0.346831\n",
      "[151]\tdtrain-F1_score:0.643665\tdval-F1_score:0.346831\n",
      "[152]\tdtrain-F1_score:0.6494\tdval-F1_score:0.34829\n",
      "[153]\tdtrain-F1_score:0.650327\tdval-F1_score:0.34891\n",
      "[154]\tdtrain-F1_score:0.651645\tdval-F1_score:0.34891\n",
      "Stopping. Best iteration:\n",
      "[144]\tdtrain-F1_score:0.630814\tdval-F1_score:0.350238\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "# the same as above, but using 5-fold CV\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# Stratified k-Fold instance\n",
    "skfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "k = 1\n",
    "val_acc = []\n",
    "val_f1 = []\n",
    "val_f1_micro = []\n",
    "val_f1_wmacro = []\n",
    "val_recall = []\n",
    "val_precision = []\n",
    "\n",
    "for train_index, val_index in skfold.split(x,y4):\n",
    "\n",
    "  train_features = x.iloc[train_index]\n",
    "  val_features = x.iloc[val_index]\n",
    "  train_labels = y4.iloc[train_index]\n",
    "  val_labels = y4.iloc[val_index]\n",
    "  val_sentences = sentence.iloc[val_index]\n",
    "  \n",
    "  dtrain = xgboost.DMatrix(train_features, label=train_labels, feature_names=feature_names)\n",
    "  dval = xgboost.DMatrix(val_features, label=val_labels, feature_names=feature_names)\n",
    "\n",
    "  xgboost_tuned_weighted = xgboost.train(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    evals=[(dtrain, 'dtrain'), (dval, 'dval')],\n",
    "    feval=f1_eval,\n",
    "    early_stopping_rounds = 10,\n",
    "    maximize=True)\n",
    "  \n",
    "  xgboost_tuned_weighted_predt = np.where(xgboost_tuned_weighted.predict(dval) > 0.5, 1, 0)\n",
    "  \n",
    "  val_sentences['pred'] = xgboost_tuned_weighted_predt\n",
    "  val_sentences_hat = val_sentences.groupby('sentence', as_index=False).agg({'pred': 'sum'})\n",
    "  val_sentences_hat['Label_bias_hat'] = val_sentences_hat['pred'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "  #PATH_sg1 = \"data/final_labels_SG1.xlsx\"\n",
    "  PATH_sg2 = \"data/final_labels_SG2.xlsx\"\n",
    "  #sentences_ground_truth = pd.read_excel(PATH_sg1)\n",
    "  sentences_ground_truth = pd.read_excel(PATH_sg2)\n",
    "  sentences_ground_truth.rename(columns={'text': 'sentence', 'label_bias': 'Label_bias'}, inplace=True)\n",
    "  sentences_ground_truth = sentences_ground_truth[sentences_ground_truth['Label_bias']!='No agreement']\n",
    "  sentences_ground_truth.replace(to_replace='Biased', value=1, inplace=True)\n",
    "  sentences_ground_truth.replace(to_replace='Non-biased', value=0, inplace=True)\n",
    "\n",
    "  sentences_merged = val_sentences_hat.merge(right=sentences_ground_truth, how='left', on='sentence')\n",
    "  sentences_merged = sentences_merged[sentences_merged['Label_bias']!=99]\n",
    "  sentences_merged = sentences_merged[sentences_merged['Label_bias'].isna() == False]\n",
    "\n",
    "  val_f1.append(metrics.f1_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat']))\n",
    "  val_f1_micro.append(metrics.f1_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat'], average='micro'))\n",
    "  val_f1_wmacro.append(metrics.f1_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat'], average='weighted'))\n",
    "  val_precision.append(metrics.precision_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat']))\n",
    "  val_recall.append(metrics.recall_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat']))\n",
    "  val_acc.append(metrics.accuracy_score(sentences_merged['Label_bias'],sentences_merged['Label_bias_hat']))\n",
    "\n",
    "  k += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 451,
     "status": "ok",
     "timestamp": 1621016262190,
     "user": {
      "displayName": "Manuel Plank",
      "photoUrl": "",
      "userId": "17725031238816109220"
     },
     "user_tz": -120
    },
    "id": "Rk7-4_glfcpv",
    "outputId": "afbb71af-8994-4e9b-c227-bb98ac345f2c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5-Fold CV Accuracy: 0.5752034584111635\n",
      "5-Fold CV Precision: 0.5850374705136172\n",
      "5-Fold CV Recall: 0.4656210665882594\n",
      "5-Fold CV F1 Score: 0.5167320797219815\n",
      "5-Fold CV Micro F1 Score: 0.5752034584111635\n",
      "5-Fold CV Weighted Macro F1 Score: 0.5692833727914473\n"
     ]
    }
   ],
   "source": [
    "print('5-Fold CV Accuracy: {}'.format(np.mean(val_acc)))\n",
    "print('5-Fold CV Precision: {}'.format(np.mean(val_precision)))\n",
    "print('5-Fold CV Recall: {}'.format(np.mean(val_recall)))\n",
    "print('5-Fold CV F1 Score: {}'.format(np.mean(val_f1)))\n",
    "print('5-Fold CV Micro F1 Score: {}'.format(np.mean(val_f1_micro)))\n",
    "print('5-Fold CV Weighted Macro F1 Score: {}'.format(np.mean(val_f1_wmacro)))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "classification_baseline.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}