tq_GBDT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load in packages and data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uproot\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "\n",
    "from functions import get_eff_faker_err, train_test_split_by_part, get_eff_faker_vs_feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take in .root into arrays\n",
    "arrays = (uproot.open(\"TTbar_PU200_D49_prompt.root\")[\"L1TrackNtuple/eventTree\"]\n",
    "                .arrays(\"*\", namedecode=\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['trk_pt', 'trk_eta', 'trk_phi', 'trk_d0', 'trk_z0', 'trk_chi2', 'trk_chi2rphi', 'trk_chi2rz', 'trk_bendchi2', 'trk_nstub', 'trk_lhits', 'trk_dhits', 'trk_seed', 'trk_hitpattern', 'trk_phiSector', 'trk_genuine', 'trk_loose', 'trk_unknown', 'trk_combinatoric', 'trk_fake', 'trk_matchtp_pdgid', 'trk_matchtp_pt', 'trk_matchtp_eta', 'trk_matchtp_phi', 'trk_matchtp_z0', 'trk_matchtp_dxy', 'tp_pt', 'tp_eta', 'tp_phi', 'tp_dxy', 'tp_d0', 'tp_z0', 'tp_d0_prod', 'tp_z0_prod', 'tp_pdgid', 'tp_nmatch', 'tp_nstub', 'tp_eventid', 'tp_charge', 'matchtrk_pt', 'matchtrk_eta', 'matchtrk_phi', 'matchtrk_z0', 'matchtrk_d0', 'matchtrk_chi2', 'matchtrk_chi2rphi', 'matchtrk_chi2rz', 'matchtrk_bendchi2', 'matchtrk_nstub', 'matchtrk_lhits', 'matchtrk_dhits', 'matchtrk_seed', 'matchtrk_hitpattern'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A look at what variables you can use as inputs to your model\n",
    "arrays.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create train and test samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select features from data and put in proper format\n",
    "# 'trk_fake' is the track quality variable, truth data\n",
    "\n",
    "features = ['trk_pt','trk_eta','trk_phi','trk_z0','trk_chi2rphi','trk_chi2rz','trk_bendchi2','trk_bendchi2']\n",
    "X = np.empty((len(arrays[features[0]].flatten()),len(features)))\n",
    "for i in range(len(features)):\n",
    "    X[:,i] = arrays[features[i]].flatten()\n",
    "\n",
    "y = arrays['trk_fake'].flatten()\n",
    "y[y==2] = 1\n",
    "\n",
    "# Grab pdgid for certain studies\n",
    "pdgid = arrays['trk_matchtp_pdgid'].flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get rid of any nan instances for training purposes (doesn't converge)\n",
    "find_nan = np.argwhere(np.isnan(X))\n",
    "X = np.delete(X, find_nan[:,0], 0)\n",
    "y = np.delete(y, find_nan[:,0])\n",
    "pdgid = np.delete(pdgid, find_nan[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create train and test sets with 2500 of each mu, elec, had, fake in train, rest in test\n",
    "X_train, y_train, pdgid_train, X_test, y_test, pdgid_test = train_test_split_by_part(X,y,pdgid,2500,2500,2500,2500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train gradient boosted decision tree (GBDT)\n",
    "### You can use either the sklearn or xgboost model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(n_estimators=500)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This creates GBDT with 500 trees that each have a max depth of 3\n",
    "# Visit https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html\n",
    "#   for more information on how to configure GBDT\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "clf_GBDT = GradientBoostingClassifier(n_estimators=500, max_depth=3)\n",
    "clf_GBDT.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster=&#39;gbtree&#39;, colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=7, gpu_id=-1,\n",
       "              importance_type=&#39;gain&#39;, interaction_constraints=&#39;&#39;,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "              min_child_weight=7, missing=nan, monotone_constraints=&#39;()&#39;,\n",
       "              n_estimators=100, n_jobs=0, num_parallel_tree=1, random_state=0,\n",
       "              reg_alpha=34, reg_lambda=1, scale_pos_weight=1, subsample=0.66,\n",
       "              tree_method=&#39;exact&#39;, validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a GBDT with 500 trees, max depth of 3 using the XGBoost sklearn API\n",
    "# https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn\n",
    "import xgboost as xgb\n",
    "clf_GBDT = xgb.XGBClassifier(n_estimators=500, max_depth=3)\n",
    "clf_GBDT.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check GBDT performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEbCAYAAAAvc3j1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XucXWV97/HPb+/Zc00mk5AQQi4kkRATEYgOESsUEdGALVihFnx5waI5VrHt0dqiWPRgT4/X2vYlHoqVop4ql9JTI0bjQblYhEgsiiQhEMIlk5BkcptkMrd9+Z0/1trJzpq9Zk+SmbVnZn/fr9d+zV5rPWut39ozs377eZ611mPujoiISDmpagcgIiJjl5KEiIjEUpIQEZFYShIiIhJLSUJERGIpSYiISCwlCRERiaUkMcGY2TIzy5vZI2WWzTczN7P2MsseNLOvReadY2Z3mdkOM+szs81mdoeZvXqUj+FCM/tVuM8tZvahYaxzsZn9wswOmtnLZvYFM6uLlDEz+3Mze9rM+sNyn4+U+YiZbTSzXjPbZGbvLbOvK81sQ7iNDWb2B5Hld4Sfc+nrsZi4zcx+HJa5KrLsRjN7xMwOmdmgG5rM7Noy+ym+zi0pV275h0qWN4YxP2lmWTN7MCbWejO72cyeD4/9JTP705LlGTO7ycyeC393vzGzFZFtvBATzw8j5WaZ2bfMrDPc1gYzu7DCMbmZ3XIsZaSyuspFZJz5IPB14L1mtsTdNx7PRszs94B7gfuB9wCbgZOAK4HPA28bmXAH7XcBsBq4HXg3cD7wdTPrdPd7Y9Y5K1znfwHvBWYDtwJp4C9Kin4F+D3gE8BvgSnArJLt/AnwBYLPcC2wHPiGme1z9x+EZV4P3AV8Bvh34B3APWb2BndfW7Kv4udWNBBzyB8H8jHLGsJ9PAh8qszyu4AfR+Z9CXgDsC4y/4PAfSXTXSXv00Af8DXgMqAtJp7vAXOBlcCzwEygqWT53xAc8weBjcBbgf9rZr/j7k+EZc4N91c0C/gVcHdxhpm1AY8A/0nwd9YJLAR2RdYr1Q78oHQ7wywjlbi7XhPkRfAPux84C/gm8OXI8vmAA+1l1n0Q+Fr4vpngH3NVzH7aRvEYvgA8G5n3z8CjQ6zzt8ATkXm/D/QCk8PpxUAWWDLEdn4BfDUy7yvAf5ZM3wX8v0iZ+4HvlUzfAdw3jGNtB7YCJ4e/l6tiyl0V/KtW3F5z+Pv/VGR+7LbLbONrwINl5r+FILFMH2Ld7cCfRebdC/yfIda5MYy5OfL7fOQY/26+AWw60TJ6DX6puWliuQp40d2fBL5DUJvIHMd23gpMJ6gxDOLu++NWNLNPmVl3hdcFQ+z79cBPIvPWAO1DHEsDwTfhUr1AI/DacPoKYAuwImzCeiFszjh5GNtZXrLvuPh+JzLvfDPbZWbPmNk3IvvBzCYTfDP/b+6+i5HxTqAF+Jcyy/7BzHab2eNm9iEzO9b//bcDjwMfM7MOM3vWzP7RzCaVlIn7/M4vt0EzM+A6giTSE9nXWguaOneZ2a/N7PqwfLntTAKuJkgCZQ2njJSnJDGxfIAgOQA8BPQAlx/HdhaFP4+nqepW4JwKr2hTSKlTgJ2ReTsJmkanx6yzBnidmb3HzOrMbDZwU7is2OSwEDiN4ERxLUGzyCuBH5ScMNcAf2xm54Z9Be0En2mmZN9x8Z1SMv1jgmaviwmak5YDPzOzhpIytwI/dvfVMcd0PFYS1GBejsy/Cfgj4M3AnQS1o3LNV0NZSHCyP5ugyfF6YAVBraloDfDnZrbYzFJmdglBc1y02afoEmABQU0xuq8PEyT1twL/QPCF5SMx23kXQYL61hDxD6eMlFPtqoxeI/MCTgdywKkl8/4n8KOS6fkMr7npr8JyU6twHM8Afx2Zd2EYzylDrPcxguaQHHAIuCFc553h8tvC6TNK1jkjnPe6cLqJoC8kG25nG0HzlwMnh2UGgPdE9v0+oG+I2E4Nt/mOcPo9wFNAY0mZE2puAl4VbuOyYXzGfwl0xSyLa276CUGtYErJvLeE+5wZTs8A/oOgjyUHbAJuAXpi9nUP8Msy8weAX0Tm/S2wMWY7jwN3VzjmimX0Kv9STWLi+ABBh+BLZpYzsxzBifItZjY3LFPsrJxSZv22kuXPhD+XHGsQI9DctIOjv5VD0GafA/bEreTufxcewzyCb/3fDxc9H/58Gci5+zMlqz0bbndeuI1ed/9jgrb9+eH8F4CDwO4K8UVrF6WxbQc6OFJDuxhYCnSX/K4A7jKz/4zbTgUrCfo3oh3Z5awFWs1s5jFs/2Vgm7uXdngXa5rFz6/T3d9O0OR1GkFNrZsjv4PDwua3Kyjf/PMysCEyb2NxP5HtnEPQtzNUU1PFMhJPSWICsOBSz/cBn+ToZp2zgSeB9wO4+z6Ck91rI+u3EtRENoWzfhKWuyFmf3FXv8CJNzc9StAsUuoSYJ27Z4dYDw9sd/de4BqCk+Z/hYsfAerM7BUlqywkaMZ6MbKdrLt3uHueoHnqPncvlMR3SZn4fhEXl5lNJ7jiqtgMdCPBxQWlnwkEV2INuuS2EjNrJKid3F4S51DOIeg7iO1bKuMR4NRIH8QZ4c/o59fn7tsIPtsrOZKwS10L9BM0f5Xb1+LIvDOi+wmtJEjk9w8R+3DKSJxqV2X0OvEXwTeyLHBSmWV/RfAPkgqnPwnsJbi89BUE7eU/CMs0RbY5APyQ4CQ4H3gN8Dngh6N4LAsImov+nqAm84EwjitLylwPPB1Z7xPAqwmaXf46XOftJctTBJdaPgQsC18PAY+VfDZnEJxsF4Wfy50EtZf5Jdv5HYLaxycJvil/Mvzsi01Wk4AvE3RwzwfeSJBYOgivtIo57kHNTQTfnM8hSB7OkYQyKVLu3QRNPPPKbPf3CS5JPTP8fX+AoMb4D5FyS8Nt30mQxM8BzilZPokg6d4TfsZvIGgyu6ekzOsI+iAWAhcAPyXoV2iL7MsIaqvfiPkszg0/0xsJvrz8YRjzRyLlmsP5Nw7xuVYso1eF/8lqB6DXCPwSYRXwk5hlC8MTzFvC6TTwUYIaRnd48rqz9ERYsu5rw5PCToJvfc8RdFS+apSP50KCGkA/QVPFhyLLP0ukjR74GcE3416CE/+lZbY7KzyegwTX3P8rYXt6uHwJ8ARBh38XQfv64jLbuQp4miARbSTsawiXNRF04O4Kl78YfmZzKxxzuSRxRzg/+npjpNxDwOqY7a4Ij+kgQfL9LfBnQF2k3Avl9hUps5igltlD0F9zCyWJL/y9bSCopewGvk1JH1lJuYvC7S8f4vN4G/CbcFvPAH8KWKTM+4n0w5XZTsUyeg39svCDFBERGUR9EiIiEktJQkREYilJiIhILCUJERGJNe6fAjt9+nSfP39+tcMQERlXfvWrX+129xmVyo37JDF//nzWrRvq3iwREYkys3I3Jw6i5iYREYmlJCEiIrGUJEREJJaShIiIxFKSEBGRWIklCTO7PRyK8KmY5RYOh7jZzJ40s9ckFZuIiJSXZE3iDoInUsa5lOARzYsInv/+vxOISUREhpDYfRLu/rCZzR+iyBXAtz14LO1jZtZmZrN88Hi9InIMCgWn4E7enUKBkvdOvuAUwiEDCh4sK7jjh99DvuBDLveSn354f+AE5bx0PcLyheBZ4cVt+eFlHFWOo8ocWcZR5YJ9FcIJB3Yf7GdaSz1h0eBnyQOvS599Xe5J2EeX9dj1y5WLzi+3n+Lbo+MYejteZubFS2Zy9tyhxgA7cWPpZrrZBIOaFHVw9Gheh5nZSoLaBvPmDRrRUOSE5PIFsnmnL5unP1dgIFdgIJ+nL1sgV3Cy+QLZXIGBsFw2XyCbL9CfK7Czq4/JjXVk8x4uL76cXN7JFYL33f05uvuytDTUHZ6fKwRlsvkC+YKTC0/uuXz4s+Ds78mSLxRoyqSDE3jxZO/BCb/48nBZvqChACYis+Dnya2NNZUkrMy8sn/h7n4bwcD2tLe367+gBgzkCvRm8/Rl8xzsy9KXLbC7ux932N7VSyaVoi+XZ/fBfnKF4LvYtn29ZNIpcoUCfdngJF888b+w5xDTmuvpzwUn993d/dSHZUf6vJpOGZm0UZdKUVf8mTJ6BnI01aeZ0pShLpUikzbSKaMunaK+LkVTyqhLBfOOvFLs7xlganM9jZkU6ZSRspLl4ftU+P7wT4PU4bKQsiPrlS5LGZgdeZ8yw4zD+yldbkAqFZkOy1Bcl+K2AY7eZio80xWnrWQdKylvJdsunhxTqSPljCP7PPyeI9uHIydVKz3NlL61QbOw0vUj5Uq3VTqvVLl9WpndV9xP3A4SMpaSRAcwt2R6DrC9SrHICSoUnK7eLF29WQ725Tg0kKO7L0dPNk/fQJ6+XP7wiXvr3h4mNdaxpfMQ9XUpDvRmOdCX47ld3bQ0pNnXM+TQ1mWlU0ZTJk13f47TTmqmsS5NYyZFQybN5MY62k+bRlfvAHOnNdNQlyadgp6BPKdOaaK+LkUmnQrK16Wpr0vREM4r/qxLG5l0ivrwhJ4Jpw+XKc5LpUilqvtPLnIixlKSWAVcb2Z3EoyV26X+iLHjQF+W3Qf72dHVx8tdfQzkC+w+2M+Bvix7D2V5dtdB9nQPUJc2XtzTc1z7OKW1kf5cnkUzJzO7rYnFMyfRm80zf3oL/dkCc6Y20VQfNLOc0tpIYyY4KU9rqaehLkVLQx2NmTSNdSnq0rq6W2QkJJYkzOx7BIPCTzezDuAzQAbA3W8FVgOXAZsJxtB9f1Kx1bKegRxb9/bywp5D9A7k2byrm1zB+a+X9rHzQB+5vLNtf2/s+k2ZNK1NdbQ2ZpjakqG5vo7zT5/OQK7AzNZGFkxvobUpQ0tDmpb6Oloa0jTUpWmqT9OUSdOgE7rImJbk1U3XVFjuwEcSCqcmFArOvp4BOvb18uLeHp7a1sULuw+xubObbL7A1r3xJ/9505ppaahjWkuGS5bOpLk+zawpjZzc2sipU5qY2pJhxuQGGurSCR6RiCRtLDU3yXHadbCPzbu6+fXW/Tzx0n627+/l2Z3dZAuFspfitTVnOG1aMxcsmkFzJs3cac3Mbmti7rRmZkxuoK0po3Z0EQGUJMaVbL7Akx1dbNjexW+3dfFkRxdP7zg4qNzstibOXTCVma2NvHr2FGa3NTHvpCARTG7MVCFyERmvlCTGsN6BPE9t7+L+DTv5+bO72fDygaOWT26sY/mCaZx56hSWzWvjtadNZdaUxqpfMiciE4eSxBjh7nTs62XVb7bz0DOd/Laji95s/qgyr5nXxtvOOpXXLzyJhTNaaMyoP0BERpeSRBUVCs6P1+/gO4++yKNb9hyenzJYNm8qZ89p47STmrlg0XQWTG9RDUFEEqckUQUH+7J8/cHn+ObPn2cgXwBg2bw2lsxq5crXzGHZ3DZ1HIvImKAkkaBfbN7NrQ9v4eFnOgGY2pzhLy9azDvPnUurOpRFZAxSkkjA0zsO8NlV63lsy14Azj99On/+5kW0z59W5chERIamJDGKtu3v5TPfX8/9G3cC8Eftc7nh0lcyNXyEsYjIWKckMUoefW4PH/jW4xwayHP52afyibcuZu605mqHJSJyTJQkRlg2X+Azq9bz3bUvMbutiX//8LksPmVytcMSETkuShIjKJsvsPLb63hgUydvXjKTz1/5aqZPaqh2WCIix01JYoT0DuT5q3uf5IFNndxw6Sv50IWvqHZIIiInTEliBOQLzrX/8kvWPr+Xj77pdCUIEZkwlCROUKHgrPz2OtY+v5dPv20JH7hgYbVDEhEZMRrt5QTd/sjz/PTpXXzkolcoQYjIhKMkcQL2dPfzxR9v4nfPmMFfvGVxtcMRERlxShLHyd155z89Sq5Q4MbLlujheyIyISlJHKe/v/9Znus8xEfftEj3QYjIhKUkcRwO9GX5559v4byF0/jwRbqSSUQmLiWJ4/CRf/0vDg3k+eSlS2io08A/IjJxKUkcozXrd/DzZ3fzjmWzOXtuW7XDEREZVUoSx8Dd+fyPnqatOcPf/MGZ1Q5HRGTUKUkcg0c27+H53Yf4xFsX01yv+xBFZOJTkhgmd+dz922gKZPmHcvmVDscEZFEKEkM09a9vWzaeZDfO2sWTfXqrBaR2qAkMUxfXPM0ANddsKDKkYiIJEdJYhjyBef+jTu5YNF0XnlKa7XDERFJjJLEMKxZv4O+bIF3LZ9X7VBERBKlJDEMv+nYD8Cblpxc5UhERJKlJFFB70Ce7619ibnTmnR3tYjUnESThJmtMLNNZrbZzG4os3yemT1gZk+Y2ZNmdlmS8ZWz9vk9HOjLsVJjRYhIDUosSZhZGrgFuBRYClxjZksjxT4N3O3uy4Crga8nFV+cJ14KmpquWDa7ypGIiCQvyZrEcmCzu29x9wHgTuCKSBkHipcPTQG2JxhfWT99eievOrWV1sZMtUMREUlckkliNrC1ZLojnFfqs8C7zawDWA18tNyGzGylma0zs3WdnZ2jESsAB/uyPLXtAOefPn3U9iEiMpYlmSTKDd3mkelrgDvcfQ5wGfAdMxsUo7vf5u7t7t4+Y8aMUQg1cP/GnQCcOXvKqO1DRGQsSzJJdABzS6bnMLg56TrgbgB3fxRoBKr2Nf6pbQcAOG/hSdUKQUSkqpJMEo8Di8xsgZnVE3RMr4qUeQm4GMDMlhAkidFrT6pg/fYuFs+czIzJDdUKQUSkqhJLEu6eA64H1gAbCa5iWm9mN5vZ5WGxjwMfNLPfAN8DrnX3aJNUUvHyZEcX7fOnVmP3IiJjQqKDIrj7aoIO6dJ5N5W83wC8IcmY4uw40EfPQJ4zZk6udigiIlWjO65j/PL5vQAsmjmpypGIiFSPkkSMHV19ACyY3lLlSEREqkdJIsaDmzqZ1lLPKa2N1Q5FRKRqlCRi7DrYx/RJ9ZiVu71DRKQ2KEnEeK7zkO6PEJGapyRRxr5DAwC0NCR68ZeIyJijJFHGtv29AMxua6pyJCIi1aUkUUbHvh4AZk9VkhCR2qYkUcbOA/0AzJvWXOVIRESqS0mijO7+HIAufxWRmqckUcZjW/bQ1pxRx7WI1DwliTKe2tZFQ50+GhERnQkjCgVnX0+WM0/VQEMiIkoSEXt7gnskXqXR6ERElCSiOvYF90gsnaVHhIuIKElEPP1yMGTp9EkajU5EREkiYnd3cI/ELN1tLSKiJBHVmEkD0FKfrnIkIiLVpyQRUbyRbpLukRARUZKIenZnN5Mb66hL66MREdGZMGLb/l6WzGqtdhgiImOCkkQJd+fXW/frwX4iIiEliRIbXz4IwKlT9GA/ERFQkjjKkx37ATjvFRq2VEQElCSOcqAvC8BZc9qqHImIyNigJFGiqzeLGTRldI+EiAgoSRzlsS17OaW1kXTKqh2KiMiYoCRRImVooCERkRJKEiUO9uVYOL2l2mGIiIwZShIlXu7qo1nPbBIROSzRJGFmK8xsk5ltNrMbYsq808w2mNl6M/tukvF19+fI5j3JXYqIjGmJNcCbWRq4BbgE6AAeN7NV7r6hpMwi4JPAG9x9n5mdnFR8vQN58gXnFTPU3CQiUpRkTWI5sNndt7j7AHAncEWkzAeBW9x9H4C770oquIPhPRInt+puaxGRoiSTxGxga8l0Rziv1BnAGWb2iJk9ZmYrym3IzFaa2TozW9fZ2TkiwfVlC8CR8SRERCTZJFHu5oNoB0AdsAh4I3AN8M9mNuj2Z3e/zd3b3b19xowZIxLclt3dQHAZrIiIBJJMEh3A3JLpOcD2MmW+7+5Zd38e2ESQNEbdof48AItOnpzE7kRExoUkk8TjwCIzW2Bm9cDVwKpImf8ALgIws+kEzU9bkghu2/4eAKY0ZZLYnYjIuJBYknD3HHA9sAbYCNzt7uvN7GYzuzwstgbYY2YbgAeAT7j7niTiy4Qj0U1u1B3XIiJFiZ4R3X01sDoy76aS9w58LHwlakvnIQCadDOdiMhhuuM6NCmsQTTU6SMRESnSGTF0oDfL5IY6zHR5k4hI0QknCTM7byQCqbat+3pJ6fpXEZGjDCtJmNkkM2uKzFtmZj8EHhmVyBLWlElpsCERkYghk4SZzTGzR4AuoMvM/s7MmszsX4C1QC9wfgJxjrqu3ixzpzVVLigiUkMqXd30eWAS8GfAleHPC4DfAovDG94mhJ6BPNNa6qsdhojImFKpueki4MPu/jWCx2QYcK+7//FEShAAT+84qLEkREQiKiWJU4DnANx9B0Hz0vdHO6hqaKlPc7AvV+0wRETGlOF0XOdL3heAvlGKpaoGcgU9t0lEJKJSn4QBD5lZ8St2E/AjMxsoLeTuZ41GcElxdw4N5KlL6xJYEZFSlZLE/4hM3ztagVRTfy4YSyJ4KoiIiBQNmSTcPZokJqT+cMChWVN0CayISKmKD/gzs9cBlwMZ4H53/8moR5Ww7oGgNS2tO65FRI4yZJIwsz8A7iHorM4BHzezj7v73ycRXFKyYXOTcoSIyNEqXd30KeAOoM3d24DPAJ8e7aCSVuyTOGlSQ5UjEREZWyolicXAF8MBgwC+BLSFo8ZNGH3Z4CpfPSZcRORolc6Kk4D9xQl37ye4oa51NINKWs9AkCT0gD8RkaMNZ2S6t5lZV8l0Cnirme0sznD3fx/xyBLUnwuSRKMeyyEicpThJIlvlpl3S8l7B8b12bXYJ6HmJhGRo1W6T6Imzppb9/YAShIiIlGVxpO43cwm/AONMungY2htylQ5EhGRsaXSV+f3ETyvaUI70JsFoLVRSUJEpFSlJFETt5d1D+SoT6do1NVNIiJHGU4j/IR/6t1ArqD+CBGRMoZzddMOs6ErFO4+rr+Cb97VTVqPCRcRGWQ4SWIlJTfUTUQNdSmNSiciUsZwksQP3H3XqEdSRXsODXD2nCnVDkNEZMyp1BA/4fsjAF7YfYh8TRypiMix0dVNQEtDHdOadfmriEiU7rgmeCzHzNbGaochIjLm1EQSqKTzYP/hu65FROSIRM+MZrbCzDaZ2WYzu2GIcleZmZtZezJxHXlcuIiIHJFYkjCzNMHTYy8FlgLXmNnSMuUmA38KrE0iLnfHHWZPnfBPHxEROWZJ1iSWA5vdfYu7DwB3AleUKfc54IsE42qPulwhuKwpowGuRUQGSTJJzAa2lkx3hPMOM7NlwFx3v2+oDZnZSjNbZ2brOjs7TyioXHjta0aP5RARGSTJM2O5r+qH704wsxTwVeDjlTbk7re5e7u7t8+YMeOEgiqOSlenmoSIyCBJJokOYG7J9Bxge8n0ZOBM4EEzewE4D1g12p3XA/lgVLrufj2WQ0QkKskk8TiwyMwWmFk9cDWwqrjQ3bvcfbq7z3f3+cBjwOXuvm40gyo2N52i+yRERAZJLEm4ew64HlgDbATudvf1ZnazmV2eVBxRxSRRp/skREQGGc4D/kaMu68GVkfm3RRT9o1JxJQtBM1NGT0qXERkkJr/+ny4JpGq+Y9CRGSQmj8zFjusB/K641pEJKrmk0TxytfWRj0FVkQkquaTRPGO63rdTCciMkjNnxmz4X0S6pMQERms5s+M+eKzm3R1k4jIIDWfJIpXN6X1WA4RkUFqPkkUm5s06JCIyGA1f2bM5tVxLSISp+bPjEc6rtXcJCISpSSh5iYRkVg1f2Ys3idRp6ubREQGqfkksb8nC+g+CRGRcmr+zLh1Xw8ArU2JPhBXRGRcqPkkcag/x2knNdNQl652KCIiY07NJ4nuvhzN9apFiIiUU/NJoj9XoLletQgRkXJqPkkM5Au6R0JEJEbNJ4lsvqC7rUVEYtT82TGXd91IJyISo+bPjr/d1oVam0REyqv5JJFJG+7VjkJEZGyq+SSRzTunz5xU7TBERMakmk4Se7r7AWhtzFQ5EhGRsammk0TPQB6ASQ26mU5EpJyaThKFsDNCHdciIuXVdJIojkrX2qTmJhGRcmo6SeQKGnBIRGQoNX12zIU1CT2WQ0SkvJpOEgMaulREZEiJnh3NbIWZbTKzzWZ2Q5nlHzOzDWb2pJn91MxOG814Ovb1AuDobjoRkXISSxJmlgZuAS4FlgLXmNnSSLEngHZ3Pwv4N+CLoxlTLqxJzJ3aPJq7EREZt5KsSSwHNrv7FncfAO4Erigt4O4PuHtPOPkYMGc0A9rTPQCgUelERGIkmSRmA1tLpjvCeXGuA35UboGZrTSzdWa2rrOz87gDagwHG2qsV5+EiEg5SZ4dy11CVLYzwMzeDbQDXyq33N1vc/d2d2+fMWPGcQdUbG7KpJQkRETKSfJ5FB3A3JLpOcD2aCEzezNwI3Chu/ePZkCHL4FN6xJYEZFykvwK/TiwyMwWmFk9cDWwqrSAmS0D/gm43N13jXZAWd1MJyIypMTOju6eA64H1gAbgbvdfb2Z3Wxml4fFvgRMAu4xs1+b2aqYzY2IbeElsLqZTkSkvEQff+ruq4HVkXk3lbx/c5LxTA4fEZ5WkhARKaum21l6B3JMacpgpiQhIlJOTSeJQwN5Wup1j4SISJyaThI9AzmaNeCQiEismk4ST207QFNGNQkRkTg1nSRe2tvDgb5stcMQERmzajZJZMO7rZfNbatyJCIiY1fNJ4lXzmqtciQiImNXDScJjUonIlJJzSaJQ/258Ge+ypGIiIxdNZsk8oWgJjFrSmOVIxERGbtqNknkwiRRX1ezH4GISEU1e4bMh0+A1XObRETi1WySKNYk1HEtIhKvdpNEeHWTahIiIvFqNkkUO641Kp2ISLyaTRIH+4JLYAdyhSpHIiIydtVskijWIFr0FFgRkVg1myQKYXOTxrcWEYlXs2fIvKvjWkSkktpNEmFNIqWhS0VEYtVskiioJiEiUlHNJonwSeGkVZMQEYlVw0kibG6q2U9ARKSymj1FFpOEmptEROLVbJLYe6gfgLBrQkREyqjZJFG8ia4pk65yJCIiY1fNJgk1N4mIVFazSaJ4CWxKSUJEJFbNJgldAisiUlntJgnXJbAiIpXU7Cmy+IA/1SREROIlmiTMbIWZbTKzzWZ2Q5nlDWZ2V7h8rZnNH61Y1HEtIlJZYknCzNLALcClwFLgGjNbGil2HbDP3U8Hvgp8YbTiUce1iEhlSdYklgOb3X2Luw8AdwJXRMoAmTpKAAAGcklEQVRcAXwrfP9vwMVmo9MelFdzk4hIRUkmidnA1pLpjnBe2TLungO6gJOiGzKzlWa2zszWdXZ2HlcwC2dM4rJXn6IxrkVEhpDk2J3lzsbRh2IMpwzufhtwG0B7e/txPVjjkqUzuWTpzONZVUSkZiRZk+gA5pZMzwG2x5UxszpgCrA3kehERGSQJJPE48AiM1tgZvXA1cCqSJlVwPvC91cBP3PXI/hERKolseYmd8+Z2fXAGiAN3O7u683sZmCdu68Cvgl8x8w2E9Qgrk4qPhERGSzJPgncfTWwOjLvppL3fcAfJhmTiIjEq9k7rkVEpDIlCRERiaUkISIisZQkREQklo33K0zNrBN48ThXnw7sHsFwxgMdc23QMdeGEznm09x9RqVC4z5JnAgzW+fu7dWOI0k65tqgY64NSRyzmptERCSWkoSIiMSq9SRxW7UDqAIdc23QMdeGUT/mmu6TEBGRodV6TUJERIagJCEiIrFqIkmY2Qoz22Rmm83shjLLG8zsrnD5WjObn3yUI2sYx/wxM9tgZk+a2U/N7LRqxDmSKh1zSbmrzMzNbNxfLjmcYzazd4a/6/Vm9t2kYxxpw/jbnmdmD5jZE+Hf92XViHOkmNntZrbLzJ6KWW5m9o/h5/Gkmb1mRANw9wn9Ings+XPAQqAe+A2wNFLmw8Ct4furgbuqHXcCx3wR0By+/5NaOOaw3GTgYeAxoL3acSfwe14EPAFMDadPrnbcCRzzbcCfhO+XAi9UO+4TPObfBV4DPBWz/DLgRwQje54HrB3J/ddCTWI5sNndt7j7AHAncEWkzBXAt8L3/wZcbGbjefDrisfs7g+4e084+RjBSIHj2XB+zwCfA74I9CUZ3CgZzjF/ELjF3fcBuPuuhGMcacM5Zgdaw/dTGDwC5rji7g8z9AidVwDf9sBjQJuZzRqp/ddCkpgNbC2Z7gjnlS3j7jmgCzgpkehGx3COudR1BN9ExrOKx2xmy4C57n5fkoGNouH8ns8AzjCzR8zsMTNbkVh0o2M4x/xZ4N1m1kEwfs1Hkwmtao71//2YJDroUJWUqxFEr/sdTpnxZNjHY2bvBtqBC0c1otE35DGbWQr4KnBtUgElYDi/5zqCJqc3EtQWf25mZ7r7/lGObbQM55ivAe5w96+Y2esJRrs8090Lox9eVYzq+asWahIdwNyS6TkMrn4eLmNmdQRV1KGqd2PdcI4ZM3szcCNwubv3JxTbaKl0zJOBM4EHzewFgrbbVeO883q4f9vfd/esuz8PbCJIGuPVcI75OuBuAHd/FGgkeBDeRDWs//fjVQtJ4nFgkZktMLN6go7pVZEyq4D3he+vAn7mYY/QOFXxmMOml38iSBDjvZ0aKhyzu3e5+3R3n+/u8wn6YS5393XVCXdEDOdv+z8ILlLAzKYTND9tSTTKkTWcY34JuBjAzJYQJInORKNM1irgveFVTucBXe7+8khtfMI3N7l7zsyuB9YQXBlxu7uvN7ObgXXuvgr4JkGVdDNBDeLq6kV84oZ5zF8CJgH3hH30L7n75VUL+gQN85gnlGEe8xrgLWa2AcgDn3D3PdWL+sQM85g/DnzDzP47QbPLteP5S5+ZfY+guXB62M/yGSAD4O63EvS7XAZsBnqA94/o/sfxZyciIqOsFpqbRETkOClJiIhILCUJERGJpSQhIiKxlCRERCSWkoSIiMRSkhA5BmZ2R/iY8ejrnMiyrJltMbMvm1lLuO78yDpd4fOUfr/axyUSR0lC5NjdD8yKvJ6KLFsIfJrgMfRfjqy/IizzOuCXwL1mdubohy1y7JQkRI5dv7vviLxykWVb3f27wL8Cb4+svycs8zTBs7MyhI/OEBlrlCRERlcv4SMUoswsQzDeA0A2sYhEjsGEf3aTyChYYWbdJdM/d/dLo4XMbDnwLuCnkUUPm1kBaCL4ovY84VNLRcYaJQmRY/cwsLJkurfkfTGB1BHUIL7P4EFv3gWsJ3gi61eBle4+nh9NLxOYkoTIsetx980xy4oJJAtsd/dyzUgd7v4s8GyYUO4xs6XuvnuU4hU5buqTEBlZPe6+2d1fjEkQR3H3h4ANwE2jH5rIsVOSEKm+rwArzWxuxZIiCVOSEKm++4AXgL+uchwig2jQIRERiaWahIiIxFKSEBGRWEoSIiISS0lCRERiKUmIiEgsJQkREYmlJCEiIrGUJEREJNb/BxwCRZueAym5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create roc curve with AUC (area under curve) value\n",
    "y_pred = clf_GBDT.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, dt = roc_curve(y_test,y_pred)\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "\n",
    "plt.plot(fpr,tpr)\n",
    "plt.xlabel('FPR',fontsize=14)\n",
    "plt.ylabel('TPR',fontsize=14)\n",
    "plt.title('AUC = '+str(auc),fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEOCAYAAAB8aOvdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNXZwPHfM5NlsoeskI2EPQn7KjsWUVSEaq1K1datvrZa+9baVrtp7WatrW9tcaF1a6tg3XGrKyAiW5RFdgIECAQCAcKShGzn/eNOwhASSEgyd5bn+/ncz8zcOXPvc0OYJ+ece84RYwxKKaVUcxx2B6CUUsp3aZJQSinVIk0SSimlWqRJQimlVIs0SSillGqRJgmllFIt0iShlFKqRZoklFJKtUiThFJKqRaF2B1AeyUlJZns7Gy7w1BKKb/y+eefHzDGJJ+tnN8niezsbAoKCuwOQyml/IqI7GhNOW1uUkop1SJNEkoppVqkSUIppVSLNEkopZRqkSYJpZRSLfJakhCRp0WkVETWtvC+iMijIlIoImtEZKi3YlNKKdU8b9YkngWmnuH9i4He7u1W4HEvxKSUUuoMvDZOwhjziYhkn6HIDOCfxlpPdamIxItIN2NMSWfEs2HZ+xxc819G90nDEeqCkHBrc4ZDSBiEuMAZ5t7f8NxlvecMP7W80++HmyilVLN86dstHdjl8brYve+0JCEit2LVNsjKyjqnkx3Z8hljdz8Fu8/p400CcraQZMKbJBWPZBQSDiERp5YNjTi9nOfxGt9zJ63QCOt5aAQ4nB1wIUopdSpfShLSzD7TXEFjzGxgNsDw4cObLXM2u3Jv4eq1w/nkrrFkxTqgthpqq6CuGmpPNHl+AupOtPDc/blT3mtyrLpqqD4GFQdOlvP8TG1VS5faeo6QMyQjl0fC8Ug0Zy3T9FgtlGmoZWmNSqmA40v/q4uBTI/XGcCezjpZfEQoIByuFrJccZ11mtYxBupqoLYSaqpOJpq6hmTimXTcyaWm0p1g3J9p+r5nImpIaieOeuz3LFsF9TXtvw5xQmgkhLo8EpFHsgl1NZN8WvHYNCmdUuNqqFmFgjT3d4ZSqj18KUnMA+4QkbnAKKC8s/ojAOIjQwE4XNEBX47tJeJuUgoDuxJWfb1HUmohkdQ1rQVVeSQz9+uaymYSmTuJVR7ySIJNjlNX3b74xWElqBDXyUQVGuF+Hul+HnFy3ynlGl57lAlxP4ZFeRzH3bynyUgFEa8lCRGZA0wCkkSkGLgPCAUwxjwBvANcAhQCFcCNnRlPY5Ko9IEk4QscDnC4vxjtUF9v1Yo8a0Gn1Yw8Ek7DVlN18nM1lVBTYe2rqTj5uqocju5176s4+ZlzSkxyarJpTECREBbpTipR1nPP5NK4LxLCoiE8+tTHsGirnCYg5WO8eXfTzLO8b4DbvRQOcRFhAJRXtPMvWNUxHA7rCzYsynvnrK9zJ5JKd7Od51ZxslbkmXBqKqH6uEcyOu7eVwEVB+HwLuv9Wo/jtJY4mySOKPfzmOaTSuO+GI/XUSfLa61HdQBfam7yqrgIH2puUvZwOK0v0/DozjuHMR4JpsJKINXHrf6h6mOnPj9xzOPxqPvxOFTsOLVM3YnWnVucJ5NMWJSVTBo2Vyy44q3mzcZ9ce4t/uTz8FgrgaugFbRJIizEQVSYk0OaJFRnEnE3Q0V23DHrapokGc+k0kySqT7mkWSOwpE9cOIIVB2xakJnjN9hJYqIeCt5RHRxbw3PEyAywXqMSrKeR6d6t0aoOlXQJgmA+MgwDmtzk/I3zlDryzgyof3Hakg4DUmjqty9HYbKwx6P7n0VB6F8l7Wv8hCYuuaPGxZtJYvoVIhOOfkY0/XUfVHJOsbHxwV1kkiICuOgJgkVzNqTcIyxkkflQSt5VJTB8QNwvBSO7oNj++BYKexbB1vnw4ny048hDohMOjVxxHgml64QlwGxaVasyus0SRzXJKHUORFxNzvFQ0KPs5evrrASyLFSdwJxJ5Gje0/u27/Jejxt3I5YtZDEXpDUG5L6nHyMzdB+k04U1EkiMSqMwtJjdoehVHAIi4SwbOiSfeZyxlhNWcf2wdESKC+2tsM7oawQ1r5i1WAahERAUi9IbJI8Ent1bF9QkArqJNElKoxD2tyklG8ROdkElpJ7+vvGwPH9cGALHNh88nH357DuNU6Z4iYuE5L7QkoepPaH9GFWrUdrHq0W1EkiISqMiuo6qmrqcIVq55lSfkHE3V+RAtljT32vphIObjs1eZRuhO2fnBw8GR4HmSMh6zzoPhbShlgj71WzgjpJJEZZA+rKjleTHm/TSGOlVMcJjYDUfGvzVFdzsrax+3PYuRQ+/sB6zxluJY2cCZA9HjKGaye5h6BOEl3cSeKQJgmlApsz9GTyGPpNa1/FQdi5BIoWQ9EimP87wFiDCHtdAH0vgd4XWgMPg1hQJwnPmoRSKshEJkC/S60NrKRRtAg2vwdb3rc6yJ1hVsLIvwL6Xty5o/N9VFAniYaaxMHjrZzmQCkVuCITIG+GtdXXQ/FyWPc6bJgHm96xJmnMm27VRLJGB828WEGdJBprEse0JqGU8uBwWB3bWefBRb+zmqXWzIW1r8HqOZA6AMZ+H/K/GvD9F0F9H1isKxSnQ/Q2WKVUyxwO6y6q6X+FuzfBZY9ad0q9egvMGmU1S9XX2x1lpwnqJOFwCF0iQ3XUtVKqdcKiYNi34LtL4ernrRUSX74JZk+ELR9aYzgCTFAnCbDGSmhzk1KqTRwOyJ0Gt30Kl8+2Jj98/mvwzxmwf7Pd0XWooE8SiVHhHDimHddKqXPgcMKgq+GOz+Hih6BkFTw+Bj75o7WoVQAI+iSRGhtO6VFNEkqpdggJg1H/YyWL3Mvg49/AC1dZc1D5OU0SsS5Kj57ABGBbolLKy6KT4cqnYdojsG0hPDnRmhbEjwV9kkiOCae6tp4jlbV2h6KUCgQiMPwmuPFda430py60Eoaf0iQREw7A/mNVNkeilAoomSPglg+tBZOev9Iaye2Hgj5JdI21Zn/cfViThFKqg8VnwY3vWFOVv3gdbP3Y7ojaLOiTRPdEa8H2nWVnWRBeKaXORWQCXP+atSjS3Otg75d2R9QmQZ8kUmLCCQ9xsKOswu5QlFKBKjIBrn/VmmH2pRug2n/+KA36JOFwCFkJkew4qElCKdWJYrrCFU9C2VZ4/+d2R9NqQZ8kwGpy2qHNTUqpzpYzAcZ8Dwqehk3v2h1Nq2iSALonRrLzYIWOlVBKdb6v/By6DoA37oDjZXZHc1aaJLCSRFVNPfuO6MhrpVQnCwk/Od/T/N/YHc1ZaZIAcpKsO5y2HThmcyRKqaCQmmcNuPv8OZ+fEFCTBNAnNQaAzXuP2hyJUipoTPwJhEbC/N/aHckZeTVJiMhUEdkkIoUick8z72eJyHwRWSkia0TkEm/ElRITTmJUGOtLjnjjdEopBVFJ1qSA61+HfevtjqZFXksSIuIEZgEXA3nATBHJa1Ls58B/jDFDgGuAx7wUG3lpsazbo0lCKeVFo2+3ahPLHrc7khZ5syYxEig0xmwzxlQDc4EZTcoYINb9PA7Y463g8tJi2bLvGNW1gbsMoVLKx0QmwICvw5qXfHZacW8miXRgl8frYvc+T/cD14lIMfAO8D3vhAZ53WKprqtn637tvFZKedGIW6C2Ela9YHckzfJmkpBm9jUdmDATeNYYkwFcAvxLRE6LUURuFZECESnYv39/hwSXnxYHoE1OSinv6jYQMkZaA+x8cKyWN5NEMZDp8TqD05uTbgb+A2CMWQK4gKSmBzLGzDbGDDfGDE9OTu6Q4HKSoogIdbJ2d3mHHE8ppVpt6DehrBD2fGF3JKfxZpJYAfQWkRwRCcPqmJ7XpMxOYDKAiORiJYmOqSqchdMh9E+P1SShlPK+3GngDIO1r9odyWm8liSMMbXAHcB7wAasu5jWicgDIjLdXeyHwLdFZDUwB7jBeHGujD6pMWzad5T6et+r8imlAlhEF+hxPqyf53NNTiHePJkx5h2sDmnPfb/0eL4eGOvNmDwN696F55ftZH3JEfqnx9kVhlIqGPW7FLa8B6XrITXf7mga6YhrD2N6Wt0fS7f5/qRbSqkA0+sC69HHVq/TJOGha5yL7omRLNt+0O5QlFLBJi4dkvrAtoV2R3IKTRJNjMxOYEXRQe2XUEp5X85E2LEYaqvtjqSRJokmRvVI5HBFDVtKdVCdUsrLekyCmgooXmF3JI00STQxKicBgGXbtV9CKeVl2eNAHLBtgd2RNNIk0URGlwjS4yNYslWThFLKyyLiIW0IbPedfglNEk2ICKN7JvLZ1jJq63SyP6WUl/WYBMUFcMI3mrw1STTjK/1SKK+sYeWuw3aHopQKNpmjwNRBySq7IwE0STRrXO8knA5h/sZSu0NRSgWbboOtx5I19sbhpkmiGbGuUIZ378L8TV6ZNkoppU6KSYWYblqT8HXn90thQ8kR9pZX2R2KUirYdBsEJavtjgLQJNGi8/umADB/kzY5KaW8rOtAOLAZairtjkSTREv6pEaTHh/BRxv22R2KUirYpOaBqYf9m+yORJNES0SEqf278snmAxw4dsLucJRSwSTFPQts6Xp740CTxBnNHJlFdV09c5bttDsUpVQwSehhLUJUusHuSDRJnEmvlGjG907iheU7dcI/pZT3OEMgsbfVL2EzTRJncfWITErKq1i4RW+HVUp5UWJPKNtqdxSaJM7mwryuJEWH8/xSbXJSSnlRQg84VAT1dbaGoUniLMJCHFw9IoOPN+5j92H7b0dTSgWJhByor4Eju20NQ5NEK8wcmYUB5i7X2oRSykviu1uPh4psDUOTRCtkdInk/L4pzF2xixqdGVYp5Q0JOdbjwe22hqFJopWuOy+L/UdP8P46HVynlPKC2AxrAaLD9rZgaJJopYl9UkiPj+DfS3fYHYpSKhg4QyAmDcp32RqGJolWcjqEa8/LYsm2MtYU6zoTSikviMuAI3tsDUGTRBtcf153ukSG8sgH9g9wUUoFgdg0KC+2NQRNEm0Q4wrllvE9mL9pP2t3l9sdjlIq0MWmwdG9YOyb8UGTRBtdP7o7sa4Q/vLRFrtDUUoFuphuUFsJVfY1cWuSaKNYVyg3js3hg/X7KCz1jYXKlVIBKrab9XikxLYQNEmcg+tHdycyzMmD7260OxSlVCCLcSeJo5ok/EpSdDh3fKUXH27Yx8LNOvGfUqqTaJLwXzePyyE7MZLfv7OBOp1GXCnVGWK6Wo/H7BvE69UkISJTRWSTiBSKyD0tlLlKRNaLyDoRecGb8bVFeIiTuy7sy8a9R/nnkiK7w1FKBaLQCAiPhWP2tVh4LUmIiBOYBVwM5AEzRSSvSZnewL3AWGNMPvC/3orvXFw2sBvjeyfxfx9u4XBFtd3hKKUCUVRy0NQkRgKFxphtxphqYC4wo0mZbwOzjDGHAIwxpV6Mr81EhJ9ekkt5ZQ3/96HeEquU6gTRqXDMvq9CbyaJdMBzEpJi9z5PfYA+IrJYRJaKyNTmDiQit4pIgYgU7N9vb8dxbrdYbhiTzbOfFbG48ICtsSilAlB0Mhz3gyQhIvkiMrCZ/QObNhu1dIhm9jXt8Q0BegOTgJnAP0Qk/rQPGTPbGDPcGDM8OTm5FafuXD+Z2o8eSVH8+OU1VFTX2h2OUiqQRCXDcf/ok5gN9G9mf577vbMpBjI9XmcATWeuKgbeMMbUGGO2A5uwkoZPiwhz8pvL+7P7cCVPLtxmdzhKqUASmQSVh6DOnj9A25IkBgLLm9m/AhjQis+vAHqLSI6IhAHXAPOalHkdOB9ARJKwmp/84lt3TM8kLspP5YmFW3UktlKq40QlWY+VB205fVuSRB0Q18z+LjTflHQKY0wtcAfwHrAB+I8xZp2IPCAi093F3gPKRGQ9MB/4kTGmrA0x2urXM/oTFR7CnXNWcqLW3sXLlVIBIjLReqyw56uwLUliIfAz962sAIhICPAz4JPWHMAY844xpo8xpqcx5rfufb80xsxzPzfGmLuMMXnGmAHGmLltiM92KbEu/njlQNaXHOH37+iUHUqpDhCZYD1W2FOTCGlD2R8DnwKFIvKpe984IBqY0NGB+avJuamNdzuN7pnIRfld7Q5JKeXPIhqShI/XJIwxm7D6JV4AEoBE4HlgkDFmQ+eE55/uvaQf+Wmx/PTVLyk9UmV3OEopf+ZHzU0YY0qMMT8zxlxqjLnEGPNzY4y9a+v5oPAQJ3++ajCVNXV865kVlFfU2B2SUspfRfpwTUJEhoqIw+N5i5t3wvUffbvG8MR1w9iy7yg3PLucqhrtyFZKnYPQCAiNtG6DtcHZahIFQJLH8xXux6bbis4K0J9N6JPMI1cPZtWuw/zgxVU6W6xS6txEdIFKe1anO1vHdQ6w3+O5aqPLBqWx70gVv3l7A498sJm7L+prd0hKKX8T0cW2cRJnTBLGmB0AIhIK3I41+d4ObwQWSG4el8P6kiP8bX4h6V0imDkyy+6QlFL+xMaaRKs6ro0xNcB3acWgOXU6EeHBKwYyqW8yP399LUu2+s34QKWUL4iI99k+CU/vAV/prEACXViIg0dnDqF7QiS3PLeC5dvtqToqpfxQeBycOGLLqduSJD4Cfici/yci14vIFZ5bZwUYSGJdobzw7fOIjwzj9he+YNfBCrtDUkr5A1csVPl+kvgbkALcCTwHvOyxvdTxoQWmrnEuHrt2KJXVddz47AoOHtcV7ZRSZxEeA9VHod77t9K3ZcS14wyb8+xHUA0GZcbzxHXD2FF2nG/8fSmV1TqGQil1BuGx1mO192eYbsuiQ98UkfBm9oeJyDc7NqzAN653Eo9fO4yNe49ywzPLOX5CFytSSrXA5U4SNjQ5taW56Rmanyo8xv2eaqML8lL5/RUDWLb9IFc89pn2USilmhceYz3a0HndliQhnL7cKEAWUN4x4QSfmSOzeObGEZSUVzJj1mKWbdPbY5VSTYT7cE1CRL4UkTVYCWKhiKzx2NYBi4APOzvQQHZ+3xRev30s8ZGhXPfUMuat1jkTlVIeXO5GnBNHvX7q1qwn8bL7sT/wNuDZc1INFAGvdGxYwadHcjSvfWcsNz+3gjvnrGRjyRF+dFFfRHT8olJBr6EmYUNz01mThDHmVwAiUgS8aIzRBRI6SVxkKP++ZRT3vbGOxxZs5cvd5fzxykF0jXPZHZpSyk4NfRJV3m/Zb8stsM8BiMiVIvITEYl3v+4pIgmdFWCwcYU6+f0VA7j/sjxWFB1k2l8XUXTguN1hKaXs5LKvJtGWW2B7ARuBJ4DfYq1OB/Ad4KGODy14ORzCDWNzePU7Y6murefKJ5awepc9k3sppXxAaCSI05Y+ibbc3fR/wPtAKlDpsX8ecH5HBqUseWmxvPKdMbhCHXz9iSX89aMtuiaFUsFIxGpy8sW7mzyMAR42xjQdHrwTSOu4kJSn3qkxvH77WCbnpvCnDzYzc/ZSSo9qt5BSQccV69vNTW6hzezTcRKdLCk6nMeuHcr9l+WxuvgwFz3yCe+v22t3WEopbwqP8/maxPvAXR6vjYjEAr/CujVWdSIRq5/i7TvHkd4lglv/9TkPv7dJm5+UChZhUVDj/ZtY2pIk7gLGicgmwAW8iDVGohtwT8eHpprTKyWGl28bw9eGZvC3+YVc8fhnrCnWTm2lAl54NJzw4Qn+jDF7gMHAg8CTQAHwY2CIMWb/mT6rOpYr1MmfrhrEX64ZzO5DlXzt8c945fNiu8NSSnWmsGio9u2aBFgT/JUDa91bGHCjiHy3owNTZzdjcDrv3DmOIZld+OFLq/nRS6t1fQqlAlVYtC1ThbdmWg4AROQ64B9YE/0d4tTJ/gzwWMeGplojJdbFP28eya/eXMeLK3Yxf1Mpf505lNE9E+0OTSnVkXy9uQlrAN1DQJQxpqsxppvHprfA2sgapT2Q1747logwJ9c/tYzfvLVeO7WVCiShEVDj/eUE2pIkYoFnjTG6Oo6PGpQZz1vfG89Xh6Tzj0+387XHP2Nnma5RoVRAcIZDfQ3U13v1tG1JEs8Dl7bnZCIyVUQ2iUihiLR4R5R7figjIsPbc75gFBcRyh+vHMifrxrE1tJjTHp4Pve9sZZD2lehlH8LcS8MWnfCu6dtQ9m7gNdFZDLwJVDj+aYx5oEzfVhEnMAsYApQDKwQkXnGmPVNysUAdwLL2hCb8iAiXDE0g1E9EnlsfiHPLdnBqyt3878X9OHaUVm4QnVJcqX8Toh7NujaKqvpyUvaUpP4H2Aq1vQclwNf99iubMXnRwKFxphtxphqYC4wo5lyv8bq+9C5J9opPT6C314+gHe/P568brH8+q31XPjIJ7yxajf12l+hlH9pqEnUercm0ZYk8Qvgh8aYFGNMf2PMAI9tYCs+nw7s8nhd7N7XSESGAJnGmLfaEJc6i9xuscy99TyeuWEEUeEhfH/uKi55dBEfbdiHMZoslPILjTUJ300STqwZX89Vc0usNX5DiYgDeAT44VkPJHKriBSISMH+/TqOrzVEhPP7pfD298bxl2sGU1VTx83PFfC1xz9jyVZdV1spn+cHNYlngGvbca5iINPjdQbguZhzDNYSqQvcq+CdB8xrrvPaGDPbGDPcGDM8OTm5HSEFH4dDmDE4nQ/umsjvrxjAnsNVzPz7Ur72+Gd8vFFrFkr5rMYk4d2W+LZ0XEcCt4jIRcAaTu+4vvMsn18B9BaRHGA3cA3wDY/PlwNJDa9FZAFwtzGmoA0xqlYKdTqYOTKLy4ek8++lO3j60+3c9GwBAzPiuPfiXB2Mp5Svsam5qS1JIhdY6X7er8l7Z/3z0xhTKyJ3AO9hNV09bYxZJyIPAAXGmPY0Zalz5Ap1csv4Hlw/ujv/WbGLv3y0hZl/X8rInARuP78XE3onIdJcS6FSyqtsqkmIvzcvDB8+3BQUaGWjo1RW1/H8sh089el2Ssqr6JEUxdUjMpk+OI1ucd677U4p1cSu5fDUFLjuFeh1QbsPJyKfG2POOhatrRP8qQAXEWbVLBb8aBIPXTmQqPAQfv/uRiY8NJ/7562jsNT7a+wqpbCt47otzU0qiISHOLlqeCZXDc9k7e5yHl+wlX8v3cGznxUxMjuBmaMymZrfjYgwHZinlFd4Dqbz5mm9ejbll/qnxzHr2qHsP3qCV74oZs7ynfzgxdX8NHQt14zM5K4pfYhxNbeyrVKqwzjDrEetSShflRwTzm0Te3Lr+B4s3V7G88t28sziIl53T/nxjVFZhDq1BVOpTmFTTUL/R6s2cziEMT2TmPWNobxx+1j6pMZw37x1TP7TQp5YuJUdZd5fPUupgNfYJ+HdyTo1Sah2GZQZz9xbz2P29cNIjgnnwXc3MvGPC7jwkYXMWb6Tyuo6u0NUKjBon4TyVyLChflduTC/K7sOVvDeur28sHwn9776Jb9+az2Tc1O5KD+V8/umEBWuv3JKnRO9u0kFgsyESG4Z34Mbx+awbHsZb60p4b9r9/Lm6j2Ehzi4IC+VC/NSmdQ3hbgI7exWqtUcTnCEaE1CBQanu99iTM8kHpiez5JtZby9poQPN5Ty9poSQhzCzeNzuG5UdzITIu0OVyn/EOLSmoQKPCFOB+N7JzO+dzL19YaCHYd46L8beXLhNp5cuI2ZIzP5ydR+xEeG2R2qUr4tJNynV6ZTqt0cDmFkTgIvf2cMuw5W8LPX1zJn+S7+U1DMJQO68cMpfchOirI7TKV8U4hLm5tU8MhMiOS5G0ewbs8R/lOwi7krdvHWmj1cN6o7v5qej8OhEwsqdYqQcG1uUsFFROifHkf/9DhuP78Xf/14C/9auoM1xYf54YV9Ga+z0Cp1ktYkVDBLjXXx6xn9yU6MYvYn2/jm08vJ6BLB14dlcuXwDNLjdRZaFeScYVqTUMFNRLhlfA+uHpHJ66v28P66vTzy4WYe+XAzY3slcsOYHCb1TdbpP1Rw0rublLLEuEK5/rzuXH9ed7btP8bba0p4ftlOvv3PAmJcIYzITmB0j0S+OiSd5Jhwu8NVyjts6JPQRYeU36ipq2fBpv18vHEfS7cdZPsBa46oiX2SuWJoOuf3SyFWZ6NVgez5q+DYPvifhe0+VGsXHdKahPIboU4HU/JSmZKXCsCWfUeZu2IXb68p4ftzVxETHsLEvslcNiiN8b2TiAzTX28VYPTuJqVar3dqDL+YlsdPL8nli52HmLN8J++t3ctba0oIczronx7LpL4pXDU8k65xLrvDVar9QsL17ial2srpEEZkJzAiO4Gqy+soKDrEJ1v2s7jwAH/+YDN//mAz3eJcfKVfCtMGpnFejwS9rVb5p5BwqPPuVOGaJFRAcYU6Gdc7iXG9kwDYvO8on245wJJtZbz8eTHPL9tJflosU/O7cvXITFJitIah/IiOk1CqY/VJjaFPagw3jcuhsrqO11ft5rnPivjTB5v5y0dbmNAnmfP7pTAkM56+XWP01lrl2/QWWKU6T0SYk5kjs5g5MovC0qO8sGwXr60s5uONpQC4Qh1c3L8blw7oxojsBOIi9U4p5WO0T0Ip7+iVEsMvL8vjF9Ny2X24koKiQ43rX7y2cjcOgdxusYzITuCyQWkMzYrXfgxlvxAX1NdCfZ21voQ3TumVsyjlo0SEjC6RZHSJ5KtD0rnvsny+2HmIZdsOsqLoIM8tKeLZz4pIj49gTM9ExvRKZEB6PD2TozRpKO9zuqfTrz0BYd5Zh0WThFIeXKHOxsWSAMqOneCdL0v4tPAA763by0ufFwPQKyWaC9zLsvZPj9O+DOUdnutca5JQyn6J0eFcPzqb60dnU1tXz5bSYyzZWsYH6/fxxMKtPLFwK2EhDgZnxjOhdxJDs7owMDOeaF3LW3UGG9a51t9kpVopxOkgt1ssud1iuWlcDqVHqli6/SBrdh3m08IDPPz+ZgAcAgMz4hma1YUL8lIYmKFJQ3UQz5qEt07ptTMpFWBSYl1MH5TG9EFpgNU0tWrXYZYXHWTljsM8v2wHTy/ejgjkJEYxoU8yVw3PJC8t1ubIld/SmoRS/isxOpzJualMzrXmljp+opbPtpaxoeQIS7eV8cKynTy3pIhxvZIY1r0Lw7snMLRg02IOAAAXrElEQVR7vM4xpVqvIUl4cZ1r/e1UqpNEhYc0Tkh45+TelFfU8NjCQj5cv49FWw4AEOIQ8tPjGJndhWHdExjfO4kobZpSLQn0moSITAX+AjiBfxhjHmzy/l3ALUAtsB+4yRizw5sxKtVZ4iJDuffiXO69OJdjJ2opKDrI8u3uW20/28HfF23H6RD6psYwrncSk/okM6ZXkt1hK18SyH0SIuIEZgFTgGJghYjMM8as9yi2EhhujKkQke8ADwFXeytGpbwlOjyESX1TmNQ3BYCqmjoWFx5g5c7DrCg6yFOfbmf2J9vIT7MG9OWnxXJej0QyE7xz26PyUY1JIjBrEiOBQmPMNgARmQvMABqThDFmvkf5pcB1XoxPKdu4Qp2n9GccO1HLU4u28+GGfbxUsItnq+sAyEyIYER2AsO6d2FC72RNGsGmsbkpAGsSQDqwy+N1MTDqDOVvBt5t7g0RuRW4FSArK6uj4lPKZ0SHh/D9C3rz/Qt6Y4xhfckRlm8/yJKtZXyy+QCvfrEbgJ7JUVw7qjsjshPo100nKAx4AV6TaG4Og2bXThWR64DhwMTm3jfGzAZmg7V8aUcFqJQvEhHy0+LIT4vjxrE5GGNYu/sIn209wD+X7OCBt6zKeFxEKJNzrSasi/t31YQRiDyn5fASbyaJYiDT43UGsKdpIRG5APgZMNEY4905cZXyAyLCgIw4BmTEceuEHuwoq2B18WE+3FDK++v28eoXu4kKczJ9cBqXDUxjaPcuuEK9Mxmc6mSB3HENrAB6i0gOsBu4BviGZwERGQI8CUw1xpR6MTal/JKIkJ0URXZSFDMGp1Nfb/jvur3MWb6TV77YzZzluwgLcTC6RyKXDUpj2sBumjD8WSDfAmuMqRWRO4D3sG6BfdoYs05EHgAKjDHzgD8C0cBL7hk2dxpjpnsrRqX8ncMhXDKgG5cM6MaRqhoWbznA0m1lfLihlLtfWs2PX17NgIx4LsxLZUR2Aj2So0iKDrc7bNVaNtQkxBj/btIfPny4KSgosDsMpXxaTV09H28sZeXOwyzZeoDVxeWN76XEhJOXFsu0gWkMztRp0H2aMfCreJh4D5x/b7sOJSKfG2OGn62cDu1UKgiEOh1clN+Vi/K7AlB6tIr1e45QWHqMdXusTvAFm/YDkBQdzqCMOCb2TWZoVhfy02I1afgKEavzWqflUEp1ppQYFyl9XY2D+YwxrNtzhC93l/PZ1jIWFx7gI/eyrnERoQzMiKNvagxDu3chJymKnKQo7duwi5fXudYkoZRCROifHkf/9DhmjszCGMPuw5XM31jK6uJyNpQc4enF2/nHp9sBCHM6mJKfyoxBaYzumUiMS9cD9xovr3OtSUIpdZqGZV2vH53N9e59x0/Usm3/cYrKjrNg034+3LCPt9eUEOoULhuYxpXDMshLiyU+MszW2AOe1iSUUr4oKjykcXzGZYPSqK6tZ/HWA7z2xW7eWlPCqyt3IwI9k6MZ4Z7VdnBmPNmJkYTowL6OozUJpZQ/CAtxcH7fFM7vm8Kh49Ws3VPOyp2HWb3rMG+uLmHOcmsWnhCHkJMUxdCsLozISWBQRhy9UqK1M/xcaU2i/WpqaiguLqaqynvZ1htcLhcZGRmEhmr7r/ItXaLCGN87mfG9kwGorzds2neUdXuOsG3/MdbuOcJ76/fyYoGVOHomR3H1iEy+OiSdlBiXnaH7H2eYJon2Ki4uJiYmhuzs7ID5a8UYQ1lZGcXFxeTk5NgdjlJn5HBI43rgDWrr6tm49yhLtpYxb/UefvfORn73zkZGZidw+dB0puZ3pUuU9mecVYhLm5vaq6qqKqASBFgdiYmJiezfv9/uUJQ6JyFOR+MdVN+e0IMNJUd4b91eXli2k3tf/ZJ7X/2S/umxTOqTwqDMePLTYkmLj7A7bN8TEg7Vx713Oq+dycsCKUE0CMRrUsGroabxva/05svd5Xy4fh8fbSzl8YVbqau3ZoIYkB7HFUPTmdAnmR5JOhIcsGoSFWXeO53XzhRknE4nAwYMaHz9+uuvU1RUxIwZM+jRowdVVVVcc8013HfffSxYsKBxf2VlJdOmTePhhx+2MXqlvMfpEAZnxjM4M567L+rL8RO17mapA7ywbCe/etOaCj2jSwRT8lKZPiiN3G6xwTuYLyQc6qq9dzqvnSnIREREsGrVqlP2FRUVMX78eN566y2OHz/O4MGDmTZtGkDj/srKSoYMGcLll1/O2LFj7QhdKVtFhYcwrHsXhnXvwu3n92JHWQUfbyzl442lPL90J88sLsIV6mBszyRG90zkvB6J9EqJDp6kobfABoeoqCiGDRvG1q1bSUlJadwfERHB4MGD2b17t43RKeUbGqZCv2lcDjeNy6H0SBXLiw6yfPtBFmza3zh1iNMhDMqI48L8rozKSQjsmkZIuN7d1JF+9eY61u850qHHzEuL5b7L8s9YprKyksGDBwOQk5PDa6+9dsr7ZWVlLF26lF/84hendEYfOnSILVu2MGHChA6NWalAkBLrYtrANKYNTANgz+FKPt9xiPUlR1i4aT8PvrsRAFeogxHZCVyY35WvDk4LrGlD9O6mwNBccxPAokWLGDJkCA6Hg3vuuYf8/HwWLFjAokWLGDhwIJs2beKee+6ha9euNkStlH9Ji48gLT6Cywal8ZOp/Sgpr2TVzsMs236Qjzbu4xevr+X+eesYkB7HeT0SmZybQn5aLJFhfvzVpzWJjnW2v/i9raHvoaX9mzdvZty4cVx++eWNNRGlVOt0i4ug24AILh7Qjfsuy6NgxyE+3ljKiu0Hmf3JVp5YuJVQpzC2VxKXDujGqJxEshIj7Q67bXTEdXDr06cP9957L3/4wx+YM2eO3eEo5bdEhBHZCYzITgDgwLETrNx5mMWFB/hg/T5+tGkNAD2So5jYJ5kJfZKZ2DsZh8PHb7MNCQdTB3W14Oz8r3BNEj7otttu4+GHH2b79u06ulqpDpIUHc6UvFSm5KXyy2l5rC85wpKtZXyyZX/jXVMxrhAGZsQxxn3n1MD0ON+bnNDZsM51JThjOv10Abl86YYNG8jNzbUpos4VyNemlF2qaur479q9LC86yBc7DrFx71EAEqPCuHhAV77SL4WROYlEh/vA39XL/w7v3A13b4HolLOXb4EuX6qUUq3kCnXy1SHpfHVIOgBlx06weGsZr6/czSuf7+bfS3fiEBicGc/YXkmM7pHIyJwEe2oZYVHWo5em5tAkoZRSTSRGhzN9UBrTB6VxoraOpdsOsmL7QRYVHmDW/EL++nEh8ZGhTO6XygW5KYzplURchJdusw11d7TXVHjldJoklFLqDMJDnEzsk8zEPsncfVFfjlbV8MnmA7y7toT31u3llS+KCQtx8JW+KYzrncSonITOXS+jsSahSUIppXxOjCuUSwd249KB3aipq+eLHYd4c80ePlxfyn/X7QWsvozpg9OYNrAbQ7O6dGzCCHd3Vlcf7bhjnoEmCaWUOkehTgejeiQyqkciv55h2FFWwdJtZbyzdi/PLC7imcVF5CRFcUFuCjeOzemYqc8bkkRVx84k0RJNEkop1QEa5pnKTorimpFZHK6o5o1Ve/h4Yyl/X7Sdpz7dzsQ+yVw5LJNJfZOJOtc7pcLdCzmd0CTht8rKypg8eTIAe/fuxel0kpxsLeu4evVqBg0aRG1tLbm5uTz33HNERkY2Ti1eW1tLTk4O//rXv4iPj7fzMpRS7RAfGca3xmTzrTHZFJYe5ZUvdvPK58XM3/QFrlAHM0dmcfO4HDK6tHHEd0NN4oR3mpt8bJRIYEhMTGTVqlWsWrWK2267jR/84AeNr6Oioli1ahVr164lLCyMJ554Ajg519PatWtJSEhg1qxZNl+FUqqj9EqJ4SdT+7H4nq/w1LeGMzInkWcWFzHuD/O59NFFfLRhH60es6bNTcFj/PjxrFmz5rT9o0ePbna/Usq/hTodTM5NZXJuKtv2H+ONVXv455Iibn6ugJykKG4am830QenERZ7hdlqHE8KivVaTCPwk8e49sPfLjj1m1wFw8YPtOkRtbS3vvvsuU6dOPWV/XV0dH330ETfffHO7jq+U8m09kqP5wZQ+fGdST95eU8JTn27nF2+s4zdvb2D6oDSuGJrB6J6JzX84PBZOlHslzsBPEj7Gc52J8ePHNyaDhv1FRUUMGzaMKVOm2BmmUspLXKFOvjYsgyuGpvPl7nKeX7qTt78s4aXPixnbK5FfTMujX9fYUz8UEQ+Vh70Sn1eThIhMBf4COIF/GGMebPJ+OPBPYBhQBlxtjClq10nb+Rd/R2tpnYmG/eXl5UybNo1Zs2Zx55132hChUsoOIsLAjHgGXhnP/dPzeXrxdv6xaBvTHv2Ub0/owS3jckiMdk/uF5kIxw94JS6vdVyLiBOYBVwM5AEzRSSvSbGbgUPGmF7AI8AfvBWfr4iLi+PRRx/l4Ycfpqamxu5wlFI2iAhzcvv5vZh3xzguzE/l8QVbGfPgxzz0340crqiG2DQ44p0ljr15d9NIoNAYs80YUw3MBWY0KTMDeM79/GVgsnTa2HbfNWTIEAYNGsTcuXPtDkUpZaPMhEgeu3YY7/9gAlPyUnlswVYu+9unHI7IgvJir0zy583mpnRgl8frYmBUS2WMMbUiUg4kAt6pV3WC+++//5TXx44da7Zc0/1vvvlmZ4WklPIzfVJj+Ns3hvKNUQe47V+f85NFtTwZZljz9hMMvPyHnXpub9YkmqsRNL0xuDVlEJFbRaRARAr279/fIcEppZSvG9Mzibm3jqb7eTP4MmoMdWkjOv2c3qxJFAOZHq8zgD0tlCkWkRAgDjjY9EDGmNnAbLAWHeqUaJVSygflpcWSN30I8K5XzufNmsQKoLeI5IhIGHANMK9JmXnAt9zPrwQ+Nv6+dJ5SSvkxr9Uk3H0MdwDvYd0C+7QxZp2IPAAUGGPmAU8B/xKRQqwaxDXtOF/nzeduE82XSilv8+o4CWPMO8A7Tfb90uN5FfD19p7H5XJRVlZGYmJiwCQKYwxlZWW4XC67Q1FKBZGAHHGdkZFBcXExgdap7XK5yMjIsDsMpVQQCcgkERoaSk5Ojt1hKKWU39OpwpVSSrVIk4RSSqkWaZJQSinVIvH32ypFZD+w4xw/noQfT/lxjvSag4Nec3BozzV3N8Ykn62Q3yeJ9hCRAmPMcLvj8Ca95uCg1xwcvHHN2tyklFKqRZoklFJKtSjYk8RsuwOwgV5zcNBrDg6dfs1B3SehlFLqzIK9JqGUUuoMgiJJiMhUEdkkIoUick8z74eLyIvu95eJSLb3o+xYrbjmu0RkvYisEZGPRKS7HXF2pLNds0e5K0XEiIjf3wnTmmsWkavc/9brROQFb8fY0Vrxu50lIvNFZKX79/sSO+LsKCLytIiUisjaFt4XEXnU/fNYIyJDOzQAY0xAb1jTkm8FegBhwGogr0mZ7wJPuJ9fA7xod9xeuObzgUj38+8EwzW7y8UAnwBLgeF2x+2Ff+fewEqgi/t1it1xe+GaZwPfcT/PA4rsjrud1zwBGAqsbeH9S7BWIBLgPGBZR54/GGoSI4FCY8w2Y0w1MBeY0aTMDOA59/OXgcni33OMn/WajTHzjTEV7pdLsVYK9Get+XcG+DXwEFDlzeA6SWuu+dvALGPMIQBjTKmXY+xorblmA8S6n8dx+gqYfsUY8wnNrNDpYQbwT2NZCsSLSLeOOn8wJIl0YJfH62L3vmbLGGNqgXIg0SvRdY7WXLOnm/HWWoid56zXLCJDgExjzFveDKwTtebfuQ/QR0QWi8hSEZnqteg6R2uu+X7gOhEpxlq/5nveCc02bf3/3iYBOVV4E83VCJre0tWaMv6k1dcjItcBw4GJnRpR5zvjNYuIA3gEuMFbAXlBa/6dQ7CanCZh1RYXiUh/Y8zhTo6ts7TmmmcCzxpj/iQio7FWu+xvjKnv/PBs0anfX8FQkygGMj1eZ3B69bOxjIiEYFVRz1S983WtuWZE5ALgZ8B0Y8wJL8XWWc52zTFAf2CBiBRhtd3O8/PO69b+br9hjKkxxmwHNmElDX/Vmmu+GfgPgDFmCeDCmuMoULXq//u5CoYksQLoLSI5IhKG1TE9r0mZecC33M+vBD427h4hP3XWa3Y3vTyJlSD8vZ0aznLNxphyY0ySMSbbGJON1Q8z3RhTYE+4HaI1v9uvY92kgIgkYTU/bfNqlB2rNde8E5gMICK5WEkisJapPNU84Jvuu5zOA8qNMSUddfCAb24yxtSKyB3Ae1h3RjxtjFknIg8ABcaYecBTWFXSQqwaxDX2Rdx+rbzmPwLRwEvuPvqdxpjptgXdTq285oDSymt+D7hQRNYDdcCPjDFl9kXdPq285h8CfxeRH2A1u9zgz3/0icgcrObCJHc/y31AKIAx5gmsfpdLgEKgArixQ8/vxz87pZRSnSwYmpuUUkqdI00SSimlWqRJQimlVIs0SSillGqRJgmllFIt0iShfJKIvCUiz3bg8e5vaRbNZspm2zlLrIgUicjdNpx3kvu6A3ngmWojTRIqWDxM66ce2QV0A1Z1XjggIjeIyLHOPIdS7RXwg+mUAjDGHANa9YVsjKkD9nZuRB1LRMLcs6Iq1aG0JqFsJyKRIvKsiBwTkX0i8tNmyoSJyB9EpFhEjovIChG5qEmZfiIyT0TK3cdaIiID3O+d0twkIgPciy0dEZGjIrJaRBqmrzituUlEJoi1IFWVO8ZH3NNCNLy/QEQeE5HficgB9yIxD7snFmzumicBzwBR7nMZEbnfo4hLRJ50x1csIj9q8nkjIreLyKsichz4nXt/noi87b6mUhGZIyJdW3PdHga5r7VCRAqkoxexUX5Fk4TyBQ8DU4CvYc25MwRroRVPz2A1F30DGIC1/sebIjIIQETSgE+xpmGYgrVIyyysqRua8wJQgrU+wRCs6aWbXWNCRNKxplJf6S57M9ZMo79vUvRaoBYYA9wB/C9wdQvn/8z9fgVW01Y398+hwQ+AL93X8QfgIbFmNPV0H9aUDAOAWWKtIfAJsNZ9XRdgTb0yzyNZtea6fw/c4z53GfC8iF+vr6Law+5Vl3QL7g3rS+wEcG2TfYexpnsG6AnUA1lNPvs68Jj7+W+BHUBYC+e5H4+VvYAjwLdaKJuNlWyGexy7EHB4lLnBHXfD6n4LgCVNjvMB8I8zXPsNwLFm9hcBc5rs2wL83OO1Af7apMwDwEdN9nVxlx3Ziuue5C57kce+se59GXb/ruhmz6Y1CWW3nljLUC5p2GGs/oMvPcoMxZozf727GemYu8P3Uvfnwfqr+FPT+nb5PwP/EJGPReRnItLvDGVzsRKA53oEn7rj7uWxb02Tz+0BUloZT1OtOVbTGWyHAROa/IwaFqNp+Dm15ro9z90w5fS5Xofyc5oklN1a04zhwPprdgQw2GPLBW5qw3EaGWPux1r/+HWs5qE1InJTC8WFlhdx8dxf08x75/p/rDXHOt7ktQN4m1N/RoOx1o94C1p93Z7nbrg+/a4IUvoPr+xWiPWldF7DDhGJwlogqMFKrC/qrsaYwibbbneZL4Bxnp3JZ2OM2WKMedQYcynWdPG3tFB0PTC6SSf0OKAa2Nra8zWjmpb7TM7FF0A+sKOZn9PRhkJtuG6lNEkoe7mblp4C/iAiU0QkH3gajy9PY8xm4HngWRG5UkR6iMhwEblbRK5wF3sMqy/jPyIyQkR6ichMERnc9JwiEiEis9yDx7JFZBTWl/76FsJ8DEgDHhORXBG5FHgQ+JsxpqIdl1+EdRfTFBFJEpHIdhwLrI76OOBFERnl/jldICKzRSTmHK77NCKSLiIbReTydsaq/IQmCeUL7gbmA6+5H9di3aXj6UasO5weAjZiNZ9MwOqsxl2jmIDVTzAfq/bxPay7jZqqw+rQfQ5rOc/XsPpE7mouOPexL8bq91iFlcTmAKfdqtsWxpjPgCfcx9oP/Lidx9uD1dFcD/wXWIeVOE64tzZddwtCgb5YyUgFAV10SCmlVIu0JqGUUqpFmiSUUkq1SJOEUkqpFmmSUEop1SJNEkoppVqkSUIppVSLNEkopZRqkSYJpZRSLdIkoZRSqkX/D4pIleVAD1E4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot TPR and FPR vs decision threshold (dt)\n",
    "plt.plot(dt[1:],fpr[1:],label='FPR')\n",
    "plt.plot(dt[1:],tpr[1:],label='TPR')\n",
    "plt.xlabel('decision thresh.',fontsize=14)\n",
    "plt.ylabel('metric',fontsize=14)\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAESCAYAAADe2fNYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAH/tJREFUeJzt3X+8VXWd7/HXm19ieRoOPzIDPYBRSuVInBysaxhd59pMKYqNkDVW9tBK++VgQt2axnKoqzezx1iKSskjS40aYybGH8PFujfF4RzBXyhJx8gDmkc6KCUoRz73j7W2Ljb7cPY+rL03Z5/38/E4D9b+ru9a+7tceN58v+vHVxGBmZlZXobUuwFmZtZYHCxmZpYrB4uZmeXKwWJmZrlysJiZWa4cLGZmlisHi5mZ5crBYmZmuXKwmJlZrobVuwH1MHbs2Jg4cWK9m2FmNqC0t7c/ExHj+qo3KINl4sSJtLW11bsZZmYDiqRN5dTzUJiZmeXKwWJmZrlysJiZWa4cLGZmlisHi5mZ5crBYmZmuXKwmJlZrhws/dS+qZurVm2kfVN3vZtiZnZAGZQPSO6v9k3dfODqu9kdMHL4EG78+AymtzTXu1lmZgcE91j6YXXHVnZHsryrZzerO7bWt0FmZgcQB0s/zJg8hpHDhzBUMHzYEGZMHlPvJpmZHTA8FNYP01uaufHjM1jdsfXlULlq1UZmTB7jITEzG/QcLP00vaWZ6S3Nvt5iZlbEQ2H7yddbzMz25GDZTzMmj2GIkmVfbzEz81DYfpve0sxPPvGOl6+3eBjMzAY7B0sFzrzmHgBuPu/4PcoL11vMzMxDYRXZvnMXm7ft8NP2Zmb74GApU/umbh59ajud3Ts467rVDhczs144WMrku7/MzMpT02CRdLKkDZI2SlpQYn2LpJWSHpB0l6QJafm7Ja3L/OyUNDtd9wNJj2fWHVuNtvvuLzOz8tTs4r2kocBVwElAJ7BG0vKIWJ+pdjmwNCJukDQLWAR8OCJWAcem+xkNbATuyGx3UUQsq2b7p7c0c9TrmnhuZw9Xzp3mi/VmZr2oZY/lOGBjRHRExIvATcCpRXWmAivT5VUl1gOcAfxHRDxftZb2omnkcMaPOtihYma2D7UMlvHAE5nPnWlZ1v3AnHT5NKBJUvGY01zgx0Vll6bDZ1dIOqjUl0s6V1KbpLaurq5+HcDN5x2/163GZma2p1oGi0qURdHn+cBMSWuBmcBmoOflHUiHAW8Fbs9ssxA4Cng7MBq4uNSXR8TiiGiNiNZx48b1+yDMzGzfavmAZCdweObzBGBLtkJEbAFOB5B0CDAnIp7NVPk74F8jYldmmyfTxRckfZ8knMzMrE5q2WNZA0yRNEnSCJIhreXZCpLGSiq0aSGwpGgf8ygaBkt7MUgSMBt4qAptNzOzMtUsWCKiB7iAZBjrEeCWiHhY0iWSTkmrnQhskPQb4FDg0sL2kiaS9Hh+WbTrGyU9CDwIjAW+XsXDMDOzPiii+DJH42ttbY22trZ6N8PMbECR1B4RrX3V85P3ZmaWKweLmZnlysFiZma5crCYmVmuHCxmZpYrB4uZmeXKwWJmZrlysJiZWa4cLGZmlisHi5mZ5crBYmZmuXKwmJlZrhwsZmaWKweLmZnlysFSRWdecw9nXnNPvZthZlZTDpYq2r5zF5u37aB9U3e9m2JmVjMOlipp39TNo09tp7N7B2ddt9rhYmaDhoOlSlZ3bGV3Ojnnrp7drO7YWt8GmZnViIOlSmZMHsMQJcvDhw1hxuQx9W2QmVmNDKt3AxrV9JZmjnpdE8/t7OHKudOY3tJc7yaZmdWEg6WKVnz2XfVugplZzXkozMzMclXTYJF0sqQNkjZKWlBifYuklZIekHSXpAlp+bslrcv87JQ0O103SdK9kh6TdLOkEbU8JjMz21PNgkXSUOAq4L3AVGCepKlF1S4HlkbEMcAlwCKAiFgVEcdGxLHALOB54I50m28CV0TEFKAbOKfqB2NmZr2qZY/lOGBjRHRExIvATcCpRXWmAivT5VUl1gOcAfxHRDwvSSRBsyxddwMwO/eWm5lZ2WoZLOOBJzKfO9OyrPuBOenyaUCTpOL7dOcCP06XxwDbIqJnH/s0M7MaqmWwqERZFH2eD8yUtBaYCWwGCqGBpMOAtwK3V7DPwrbnSmqT1NbV1VVp283MrEy1DJZO4PDM5wnAlmyFiNgSEadHxDTgS2nZs5kqfwf8a0TsSj8/A4ySVLhteq99Zva9OCJaI6J13Lhx+380ZmZWUi2DZQ0wJb2LawTJkNbybAVJYyUV2rQQWFK0j3m8MgxGRATJtZgz0qKzgZ9Xoe1mZlammgVLeh3kApJhrEeAWyLiYUmXSDolrXYisEHSb4BDgUsL20uaSNLj+WXRri8GLpS0keSay/VVPAwzM+uDkn/0Dy6tra3R1tZW72aYmQ0oktojorWven7y3szMcuVgMTOzXDlYzMwsVw4WMzPLlYPFzMxy5WAxM7NcOVjMzCxXDhYzM8uVg8XMzHLlYDEzs1w5WMzMLFcOFjMzy5WDxczMcuVgMTOzXDlYzMwsVw4WMzPLVUXBIunYajXEzMwaQ6U9lvsktUv6pKS/qEqLzMxsQKs0WN4E3AF8CXhS0g8lvTv/ZpmZ2UBVUbBExGMRsRA4AvgAMBK4TdJvJX1J0oRqNNLMzAaOfl28j4jdEfEL4EPAAmA88DWgQ9JNksbn2EYzMxtA+hUsko6TdDXwJPA54BvAROAEYDRwa14NNDOzgWVYJZUlXQh8DJgC/AI4C7gtInanVX4v6VPAo7m20szMBoxKeyyfBH4IHBERp0fEikyoFPweOKfUxpJOlrRB0kZJC0qsb5G0UtIDku7KXrORdISkOyQ9Imm9pIlp+Q8kPS5pXfrjW6LNzOqooh5LREwpo86LwA3F5ZKGAlcBJwGdwBpJyyNifaba5cDSiLhB0ixgEfDhdN1S4NKIuFPSIUA20C6KiGWVHIuZmVVHpQ9IXiDpQyXKP5QOge3LccDGiOhIw+cm4NSiOlOBlenyqsJ6SVOBYRFxJ0BE/Ckinq+k7WZmVhuVDoV9DniiRPnvgM/3se34om0707Ks+4E56fJpQJOkMcAbgW2SfiZpraTL0h5QwaXp8NkVkg4q81jMzKwKKg2WCcCmEuWd6bp9UYmyKPo8H5gpaS0wE9gM9JAM2Z2Qrn87MBn4SLrNQuCotHw0cHHJL5fOldQmqa2rq6uPppqZWX9VGixPAaUujr8NeKaPbTuBwzOfJwBbshUiYkt6U8A0kqf7iYhn023XpsNoPSS3M78tXf9kJF4Avk8y5LaXiFgcEa0R0Tpu3Li+jtPMzPqp0mD5EfAdSSdJGp7+/DXwbeDGPrZdA0yRNEnSCGAusDxbQdJYSYU2LQSWZLZtllRIhFnA+nSbw9I/BcwGHqrwmMzMGt6Z19zDmdfcU5PvquiuMOAfgUnA7cBLadkQ4CfAl/e1YUT0SLog3XYosCQiHpZ0CdAWEcuBE4FFkgL4FXB+uu1LkuYDK9MAaQeuTXd9Yxo4AtYBn6jwmMzMLEeV3m68C5gn6SvAtLT4vojYWOb2K4AVRWVfySwvA0reNpzeEXZMifJZ5bXezMxqodIeC5C8jBJ4LOe2mJlZA6g4WCSdCbwHeC1F12gi4pSc2mVmZgNUpe8Ku4zkWZZVJHd0Fd8ubGZmg1ylPZa/B+b59SlmZtabSm83HkJy55WZmVlJlQbLYpLJvczMzEqqdChsFPBBSScBDwC7sisj4jN5NczMzAamSoNlKq8MhR1VtM4X8itUeAr25vOOr3NLzMzyU+kDku+uVkPMzKwx9HfO+7GS/sqvqDczs2KVTvTVJOknwNPA3aTzqUi6WtJX82+emZkNNJX2WL4JvJ7klfU7MuX/TjIxl1Vg+85dbN62g/ZN3fVuiplZbioNllOAz0XEOva8WP8IyeRbVqb2Td08+tR2Ort3cNZ1qx0uNqjU8hXuVnuVBkszsLVEeROvvEbfyrC6Yyu702je1bOb1R2l/rOamQ08lQbLGpJeS0Gh13IeyTUXK9OMyWMYkk7WPHzYEGZMHlPfBpmZ5aTS51i+CNwu6c3pthemy8cB78q7cY1sekszR72uied29nDl3GlMb2mud5PMzHJRUY8lIu4G3gGMAH5L8vr8LcDxEXFf/s1rbE0jhzN+1MEOFTNrKBXPxxIRDwJnV6EtZmbWAPoMFkmjI+KPheV91S3Us/L4VS5m1ojK6bF0STosIp4GnqH0O8GUlg/Ns3FmZjbwlBMss4BCT8TvCjMzs33qM1gi4pells3MzEqp9F1hH5B0aonyUyWdkV+zzMxsoKr0AcmvAjtLlP85XbdPkk6WtEHSRkkLSqxvkbRS0gOS7pI0IbPuCEl3SHpE0npJE9PySZLulfSYpJsljajwmMzMLEeVBstkYEOJ8o308a4wSUOBq4D3kkwYNk/S1KJqlwNLI+IY4BJgUWbdUuCyiDia5IHMp9PybwJXRMQUoBs4p6IjMjOzXFUaLN3AlBLlbwS297HtccDGiOiIiBeBm4DiYbWpwMp0eVVhfRpAwyLiToCI+FNEPC9JJDcXLEu3uQGYXdkhmZlZnioNlp8DV0h6Y6FA0puAbwG39rHteOCJzOfOtCzrfmBOunwa0CRpDElwbZP0M0lrJV2W9oDGANsiomcf+zQzsxqqNFi+ADwLrJf0hKQngIeB54CL+thWJcqKn4mZD8yUtBaYCWwGekjuXjshXf92kmG3j5S5z+TLpXMltUlq6+rq6qOpZmbWX5XOeb8deKekk4BjSX6x3wesjIiSv9AzOoHDM58nkLxnLLv/LcDpAJIOAeZExLOSOoG1EdGRrrsVmAEsAUZJGpb2WvbaZ2bfi4HFAK2trX211QaIwpwefouB2YGj4neFAaTXOu6scLM1wBRJk0h6InOBD2YrSBoL/DEidgMLSYKjsG2zpHER0UVyXaUtIkLSKuAMkms2Z5MM15mZWZ2U866wC4HvRsTOdLlXEfGtfazrkXQBcDvJq1+WRMTDki4hCYnlwInAIkkB/Ao4P932JUnzgZXpBft24Np01xcDN0n6OrAWuL6vYzIzs+opp8fyaZK7rXamy70Jkov4vVeIWAGsKCr7SmZ5Ga/c4VW87Z3AMSXKO0juODMzswNAOa90mVRq2czMrJQ+7wqT9JKk16bLSyQ1Vb9ZZmY2UJVzu/EO4JB0+WxgZPWaY2ZmA10511juBm6V1E5ye/F3JO0oVTEiPpZn48zMbOApJ1g+TPJg4hvSz2OAF6rWIjMzG9DKuXj/B9Kn6iU9DsyLiK3VbpiZmQ1MFV28J3kx5IvVbZKZmQ1kvnhvZma58sV7MzPLVbkX7y8CjiR5ut4X783MrFflXryfD754bwee7Tt38dzOHto3dTO9pbnezbEy+bw1trLmY5F0t6RRETEpIrZKWiRpdGb9WEm/r14zzfbWvqmbR5/aTmf3Ds66bjXtm7rr3SQrg89bfWzfuYvN23bU5L93uRN9zQBGZD6fD4zKfB5KMheKWc2s7tjK7nRmnV09u1nd4Y70QODzVnu1DvNKZ5AsKHvmRrNqmTF5DEPSv4nDhw1hxuQx9W2QlcXnrfZqHeb9mujL7EAwvaWZo17XxHM7e7hy7jSP1Q8QPm+1Vwjz3VGbMC83WIK9eyTuoVjdNY0cTtPI4f7lNMD4vNVWrcO83GAR8ENJhduMRwLXSno+/XxQ7i0zM7Pc1DLMyw2WG4o+/7BEnaX72RYzM2sAZQVLRHy02g0xM7PG0N+7wsysn8685h7OvOaeejfDrGocLGZmlisHi5mZ5crBYmZmuappsEg6WdIGSRslLSixvkXSSkkPSLpL0oTMupckrUt/lmfKfyDp8cy6Y2t1PGZmtreaPXkvaShwFXAS0AmskbQ8ItZnql0OLI2IGyTNAhaRvLYfYEdE9BYaF0XEsmq13czMylfLHstxwMaI6IiIF4GbgFOL6kwFVqbLq0qsNzOzA1wtg2U88ETmc2dalnU/MCddPg1oklR4qc1ISW2SVkuaXbTdpenw2RWS/BYAM7M6qmWwlPNG5PnATElrgZnAZqAnXXdERLQCHwS+LenItHwhcBTwdmA0cHHJL5fOTYOpraura/+OxMzMelXLtxt3AodnPk8AtmQrRMQW4HQASYcAcyLi2cw6IqJD0l3ANOC3EfFkuvkLkr5POttlsYhYDCwGaG1t9Qs0G8TN5x1f7yaYWZFa9ljWAFMkTZI0ApgLLM9WSGeiLLRpIbAkLW8uDHFJGgu8E1iffj4s/VPAbOChGhyLmZn1omY9lojokXQBcDvJjJNLIuJhSZcAbRGxHDgRWCQpgF+RzFQJcDRwjaTdJGH4jczdZDdKGkcy1LYO+EStjsnMzPZW04m+ImIFsKKo7CuZ5WXAXrcNR8TdwFt72eesnJtpZmb7wU/em5lZrhwsZmaWKweLmZnlysFiZma5crCYmVmuHCxmZpYrB4vtF0+za2bFHCxmZpYrB4tZjW3fuYvN23bQvqm73k0xqwoHi1kNtW/q5tGnttPZvYOzrlvtcLGG5GAxq6HVHVvZnb5be1fPblZ3bK1vg8yqwMFiVkMzJo9hSDoz0fBhQ5gxecy+NzAbgGr6Ekqrj8JdW567pP6mtzRz1OuaeG5nD1fOncb0luZ6N6ku/HexsTlYzGqsaeRwmkYOH7ShYo3PQ2FmZpYrB4uZmeXKwWJmZrlysJiZWa4cLLZf/BS5mRVzsFi/+SlyMyvFwTIIVKtX4afIzawUB0uDq2avwk+Rm1kpDpYGV81eReEp8gnNB3Pjx2f4gT8zA2ocLJJOlrRB0kZJC0qsb5G0UtIDku6SNCGz7iVJ69Kf5ZnySZLulfSYpJsljajV8QwE1e5VNI0czvhRBztUzOxlNQsWSUOBq4D3AlOBeZKmFlW7HFgaEccAlwCLMut2RMSx6c8pmfJvAldExBSgGzinagcxALlXYWa1Vssey3HAxojoiIgXgZuAU4vqTAVWpsurSqzfgyQBs4BladENwOzcWtwg3Ksws1qqZbCMB57IfO5My7LuB+aky6cBTZIKYzcjJbVJWi2pEB5jgG0R0bOPfZqZWQ3VMlhUoiyKPs8HZkpaC8wENgOF0DgiIlqBDwLflnRkmftMvlw6Nw2mtq6urn4dgJmZ9a2Wr83vBA7PfJ4AbMlWiIgtwOkAkg4B5kTEs5l1RESHpLuAacBPgVGShqW9lr32mdn3YmAxQGtra8nwMTNrVLWcA6eWPZY1wJT0Lq4RwFxgebaCpLGSCm1aCCxJy5slHVSoA7wTWB8RQXIt5ox0m7OBn1f9SMzMrFc1C5a0R3EBcDvwCHBLRDws6RJJhbu8TgQ2SPoNcChwaVp+NNAm6X6SIPlGRKxP110MXChpI8k1l+trckBmZlZSTWeQjIgVwIqisq9klpfxyh1e2Tp3A2/tZZ8dJHecmZnZAcBP3puZWa485/0AceY19wD9uwBXzYt2tbwgaGYDg4PFrMYcxtboPBRmZma5crCYmVmuHCxmZpYrB4uZmeXKwWJmZrlysAwQ1Zq33swsbw6WAaCa89abmeXNwTIAVHPeejOzvDlYBoBqz1tvZpYnP3k/ABTmrX9uZw9Xzp3mKYbN7IDmYBkgmkYOp2nkcIeKmR3wPBRmZma5crCYmVmuHCxmZpYrB4uZmeXKF+8HCM/hYWYDhXssZmaWKweLmZnlysFiZma5crCYmVmuahoskk6WtEHSRkkLSqxvkbRS0gOS7pI0oWj9ayRtlvQvmbK70n2uS39eW4tjMTOz0moWLJKGAlcB7wWmAvMkTS2qdjmwNCKOAS4BFhWt/xrwyxK7Pysijk1/ns656WZmVoFa9liOAzZGREdEvAjcBJxaVGcqsDJdXpVdL2k6cChwRw3aamZm/VTLYBkPPJH53JmWZd0PzEmXTwOaJI2RNAT438BFvez7++kw2JclqVQFSedKapPU1tXV1f+jMDOzfaplsJT6hR9Fn+cDMyWtBWYCm4Ee4FPAioh4gr2dFRFvBU5Ifz5c6ssjYnFEtEZE67hx4/p7DGZm1odaPnnfCRye+TwB2JKtEBFbgNMBJB0CzImIZyUdD5wg6VPAIcAISX+KiAURsTnddrukH5EMuS3dV0Pa29ufkbSpj/aOBZ4p//Aaho97cPFxDy77e9wt5VSqZbCsAaZImkTSE5kLfDBbQdJY4I8RsRtYCCwBiIizMnU+ArRGxAJJw4BREfGMpOHA+4D/7KshEdFnl0VSW0S0lntwjcLHPbj4uAeXWh13zYbCIqIHuAC4HXgEuCUiHpZ0iaRT0monAhsk/YbkQv2lfez2IOB2SQ8A60gC69pqtN/MzMqjiOLLHAb+F02921FrPu7BxcddXX7yvneL692AOvFxDy4+7sGlJsftHouZmeXKPRYzM8uVg6VIX+8zaxSSDpe0StIjkh6W9Nm0fLSkOyU9lv7ZXO+2VoOkoZLWSvr39PMkSfemx32zpBH1bmM1SBolaZmkR9Nzf/xgOOeSPp/+PX9I0o8ljWzEcy5piaSnJT2UKSt5fpX4Tvq77gFJb8urHQ6WjDLfZ9YoeoB/iIijgRnA+emxLgBWRsQUktfrNGq4fpbk7sSCbwJXpMfdDZxTl1ZV35XAbRFxFPCXJP8NGvqcSxoPfIbkMYW3AENJHndoxHP+A+DkorLezu97gSnpz7nA9/JqhINlT+W8z6whRMSTEXFfuryd5BfMeJLjvSGtdgMwuz4trJ70rdl/C1yXfhYwC1iWVmnU434N8C7geoCIeDEitjEIzjnJM3sHp8++vQp4kgY85xHxK+CPRcW9nd9TSV76GxGxGhgl6bA82uFg2VM57zNrOJImAtOAe4FDI+JJSMIHaMRpCL4NfAHYnX4eA2xLn7WCxj3vk4EuknfrrZV0naRX0+DnPH07x+XA70kC5VmgncFxzqH381u133cOlj2V8z6zhpK+OuenwOci4rl6t6faJL0PeDoi2rPFJao24nkfBrwN+F5ETAP+TIMNe5WSXlM4FZgEvB54NckwULFGPOf7UrW/9w6WPfX5PrNGkr4G56fAjRHxs7T4D4XucPpno81v807gFEm/IxnqnEXSgxmVDpNA4573TqAzIu5NPy8jCZpGP+f/HXg8IroiYhfwM+AdDI5zDr2f36r9vnOw7Onl95mld4jMBZbXuU1VkV5XuB54JCK+lVm1HDg7XT4b+Hmt21ZNEbEwIiZExESS8/t/0nfRrQLOSKs13HEDRMRTwBOS3pQWvQdYT4Ofc5IhsBmSXpX+vS8cd8Of81Rv53c58Pfp3WEzgGcLQ2b7yw9IFpH0NyT/gh0KLImIvt5XNiBJ+m/A/wUe5JVrDV8kuc5yC3AEyf+QH4iI4ouBDUHSicD8iHifpMkkPZjRwFrgQxHxQj3bVw2SjiW5aWEE0AF8lOQfmA19ziX9E3Amyd2Qa4GPk1xPaKhzLunHJO9cHAv8AfhH4FZKnN80ZP+F5C6y54GPRkRbLu1wsJiZWZ48FGZmZrlysJiZWa4cLGZmlisHi5mZ5crBYmZmuXKwmJlZrhwsZmaWKweLWRVIapb0B0lH1rstlUjnarmw3u2wgc3BYlYdXwRWRMRvs4WSDpV0RTrp0s50Uqa7JX06fSFonyT9m6T/7GXd0ZJC0klF5VdLuqKM3f8T8D8l/UU5bTErxcFiljNJryJ5Zcj1ReUTgftIXqHxZZIXQM4ieaX7e4BTyvyK64BZ6f6KnQNsIpnQqfC9At5PGe/CiogHSV718qEy22K2FweLGSDprvRf9VdK6k5/LpPUn/9H/obk/Wu/Lir/XlreGhE3RcT6iHgoIn4WEbOBH2faI0lfkPRbSTskPSip8Mv+FyTvgfpo0TEMBz5M8o673ZlVbwdGAv8vs+9/SHtNL0jqlLQoU385MK8fx20GOFjMss4i+X/ieOA8kulaP9eP/ZwAtEfmRXySRgP/A7gqIv5caqPY88V9XyfpfZxPMk32IuAaSX+bTk51A/CRouB7P8nLB79ftOvZwC8yk1r9M0mPaRHwZuAD7Dnh038Bx0k6uPxDNnuFg8XsFU8Cn4mIRyPiFuAy4EIASZ+QtC7tOexKl9dJ+nyJ/bSk+8qaQjKx0oZsYdpb+FP6c3Va9ur0ez8eEbdFxOMR8SPgWpKggWSY7QiSuUYKzgHuiIhsSEAyydWt6b4PAT4PLIiIJRGxMSLuiYjvZupvAYaTTIplVrFhfVcxGzRWF/Ua7gG+Juk1EXE1cLWkN5PMEz59H/s5mGSoqhwnkEzRsJhkuAqSHspI4DZJ2fYMB34HEBGPSfoV8DHgDkmvJ+kRnZnduaQ3kExJfHtm3weRuQZTwo7McZhVzMFiVpm3AA/1UecZoLmobCPJtK9HZQsj4nEASc9nigsjCe8nmT8ja1dm+Trg2nSY7SPAH9l7YrrZwMrM8Fup6WiLjU7/7CqjrtlePBRm9oq/Su+gKpgBbImI5zJl5QTLWpKewcsiYitwB3BBGbcVrwdeAFrSoarsz6ZMvWXATpI7uD5G0pPaVbSvl4fBivb9nn18/1tIjrvcXpfZHhwsZq94PfBtSW+SdAZwEVD87Ec5wXI7cLSkMUXlnyL5f65d0jxJUyW9UdI84C+BlwAiYjvJLciXS/qYpDdIOja9znNuYWcRsQP4EfBV4Ej2vr15HEk4/ltmm+3AlcAiSR+VdKSk4yR9MrPpCcBtfRyjWa88FGb2ihtJrnfcSzJsdT39CJaIeFDSfwFzgasy5R2SpgELga8Bh5MMbT0CfJdkmtiCL5Ncp5lPcpvyc8A64H8Vfd11wCeBuyPikaJ17wfWlOh5LAS60++YkH7PUgBJI4HTSK7XmPWLpyY2I3mOBXgoIi7YR52Dgc0RMbq3Opm6J5P0DKZGxEu5NbQCkn4O/DoiisNoX9ucD5waEX9dvZZZo/NQmFn5jibpXfQpIm4j6a1MqGqL9u3XZB66LNMu4NNVaIsNIu6xmFFej8XMyuNgMTOzXHkozMzMcuVgMTOzXDlYzMwsVw4WMzPLlYPFzMxy5WAxM7NcOVjMzCxXDhYzM8vV/wejqHvWCZX/dgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAESCAYAAAAFYll6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHCtJREFUeJzt3X2UHXWd5/H3J6FDIkYJoUckz8Eg4MMQ0wOND4yDoGHUhD1HlzCwiwgbcGBAwQcYBdewIIszIuMGAQMOMwMGZFltNRKEAWdUGtINjJAAQ2gIaQISQkOiJJAm3/2jqpPKzU26Kun7kHs/r3P69K1fPdzvPdXdn66qX9VPEYGZmVkRw2pdgJmZ7X4cHmZmVpjDw8zMCnN4mJlZYQ4PMzMrzOFhZmaFOTzMzKwwh4eZmRXm8DAzs8L2qHUBlbLvvvvG5MmTa12Gmdlupbu7+8WIaB1suYYNj8mTJ9PV1VXrMszMdiuSVuRZzqetzMysMIeHmZkV5vAwM7PCHB5mZlaYw8PMzApzeJiZWWEODzMzK8zhYWZmhTk8yjj+mns5/pp7a12GmVndcniYmVlhDg8zMyvM4WFmZoU5PMzMrDCHh5mZFVbV8JA0U9LjkpZLOn8Hy31KUkhqy7RdkK73uKSPVadiMzMrp2rjeUgaDswHjgF6gSWSOiJiWclyo4GzgfsybYcAc4B3AfsDd0o6MCLeqFb9Zma2RTWPPA4DlkdET0S8DiwEZpdZ7mLgcmBDpm02sDAiXouIp4Dl6fbMzKwGqhke44CVmenetG0zSdOBCRHxs6LrmplZ9VQzPFSmLTbPlIYBVwDnFV03s425krokda1evXqnCzUzsx2rZnj0AhMy0+OBVZnp0cC7gXskPQ20Ax3pRfPB1gUgIq6NiLaIaGttHXT8djMz20nVDI8lwDRJUySNILkA3jEwMyJeiYh9I2JyREwGOoFZEdGVLjdH0p6SpgDTgPurWLuZmWVUrbdVRPRLOgtYDAwHro+IpZLmAV0R0bGDdZdKugVYBvQDZ7qnlZlZ7VQtPAAiYhGwqKTtou0s++GS6UuASypWXMa6DRtZu6Gf7hV9zJg0phpvaWa2W/Ed5iW6V/Tx2PPr6O1bz4kLOule0VfrkszM6o7Do0Rnzxo2pf24NvZvorNnTW0LMjOrQw6PEu1TxzIs7Rjcsscw2qeOrW1BZmZ1qKrXPHYHMyaN4aD9RrN2Qz9Xzpnuax5mZmU4PMoYPbKF0SNbHBxmZtvh01ZmZlaYw8PMzApzeJiZWWEODzMzK8zhYWZmhTk8zMysMIeHmZkV5vAwM7PCHB5mZlaYw8PMzApzeJiZWWF+tlUZN59+RK1LMDOraz7yMDOzwhweZmZWmMPDzMwKc3iYmVlhDg8zMyvM4WFmZoU5PMzMrDCHh5mZFebwMDOzwhweZmZWWFXDQ9JMSY9LWi7p/DLzz5D0sKSHJP1a0iFp+2RJ69P2hyRdXc26zcxsa1V7tpWk4cB84BigF1giqSMilmUWuykirk6XnwV8G5iZznsyIg6tVr1mZrZ91TzyOAxYHhE9EfE6sBCYnV0gItZmJvcCoor1mZlZTtUMj3HAysx0b9q2FUlnSnoSuBw4OzNriqQHJf1K0ocqW6qZme1INcNDZdq2ObKIiPkRcQDwFeBrafNzwMSImA6cC9wk6S3bvIE0V1KXpK7Vq1cPYelmZpZVzfDoBSZkpscDq3aw/ELgOICIeC0i1qSvu4EngQNLV4iIayOiLSLaWltbh6xwMzPbWjXDYwkwTdIUSSOAOUBHdgFJ0zKTHweeSNtb0wvuSJoKTAN6qlK1mZlto2q9rSKiX9JZwGJgOHB9RCyVNA/oiogO4CxJRwMbgT7g5HT1I4F5kvqBN4AzIuKlatVuZmZbU0Rjdmhqa2uLrq6uWpdhZrZbkdQdEW2DLec7zM3MrDCHh5mZFebwMDOzwhweZmZWmMPDzMwKc3iYmVlhDg8zMyvM4WFmZoU5PMzMrDCHh5mZFebwMDOzwhweZmZWmMPDzMwKc3iYmVlhDg8zMyvM4WFmZoU5PMzMrLDc4ZGOI/55Sd+VNDZta5c0qXLlmZlZPcoVHpKmA48DpwKnA29NZx0LXFqZ0szMrF7lPfL4e+CqiHgP8Fqm/Xbgg0NelZmZ1bW84TED+EGZ9lXA24auHDMz2x3kDY8NwFvKtL8TWD105dSX46+5l+OvubfWZZiZ1Z284fFT4CJJLel0SJoIXAbcVpHKzMysbuUNj/OA/YAXgFHAr4DlwKvAVytTWu2t27CRZ19eT/eKvlqXYmZWV/bIs1BEvCLp/cAxwPtIQucBYHFERAXrq5nuFX089vw6NgWcuKCTG09rZ8akMbUuy8ysLuTtqvtXQEtE3BERl0XEpRFxO9CSzms4nT1r2JTG4sb+TXT2rKltQWZmdSTvaat/BvYu0z46nddw2qeOZZiS1y17DKN96tjaFmRmVkdynbYCBJQ7PTUeWDt05dSPGZPGcNB+o1m7oZ8r50z3KSszs4wdhoekB0lCI4C7JPVnZg8HpgB35H0zSTOBK9N1F0TEZSXzzwDOBN4A/gDMjYhl6bwLSO5wfwM4OyIW533fnbXonCMr/RZmZrulwY48fpZ+PxS4E/hjZt7rwNPAj/K8kaThwHySi+69wBJJHQPhkLopIq5Ol58FfBuYKekQYA7wLmB/4E5JB0bEG3ne28zMhtYOwyMiLgSQ9DRwY0Rs2IX3OgxYHhE96TYXArOBzeEREdlTYHux5VTZbGBhRLwGPCVpebo938FnZlYDebvqXjcE7zUOWJmZ7gUOL11I0pnAucAI4KjMup0l644rs+5cYC7AxIkTh6BkMzMrJ29X3RZJF0paJukPkl7PfuV8L5Vp2+YifETMj4gDgK8AXyu47rUR0RYRba2trTnLMjOzovJ21Z0H/A+SaxbDSe4qXwC8ApyTcxu9wITM9HiSBytuz0LguJ1c18zMKihveBwPnB4R84F+4LaI+GvgG8Bf5NzGEmCapCmSRpBcAO/ILiBpWmby48AT6esOYI6kPSVNAaYB9+d8XzMzG2J57/PYD1iavv4DW24YXARckmcDEdEv6SxgMcnRy/URsVTSPKArIjqAsyQdDWwE+oCT03WXSrqF5OJ6P3Cme1qZmdVO3vBYCbwdeAZ4kqS7bTdJj6fcPbAiYhFJ4GTbLsq83u4psIi4hJxBZWZmlZX3tFUHSWAAfBe4WNITwA2UHyTKbMh4XBWz+pO3q+6XMq9vlvQs8H7gPyPix5UqzszM6tOg4ZEOAPWPwEUR8SRARPwa+HVlSzMzs3o16GmriNhI0vOpIcftMDOz4vJe8/h/bLnnwszMmlze3lZPAhemowl2s/UDEomIfxjqwszMrH7lDY/TSe7vOJxtn0cVgMPDzKyJ5O1tNWHwpczMrFnkveZhZma2mcPDzMwKc3iYmVlhDg+re+s2bOTZl9fTvaKv1qWYWcrhYXWte0Ufjz2/jt6+9Zy4oNMBYlYncoeHpFZJn5f0XUlj07Z2SZMqV541u86eNWxKn22wsX8TnT1raluQmQH5h6GdDjwOnEpyz8db01nHApdWpjQzaJ86lmHpIMQtewyjferY2hZkZkD+I4+/B66KiPcAr2Xabwc+OORVmaVmTBrDQfuNZvyYUdx4WjszJo2pdUlmRv47zGeQjGFeahXwtqErx2xbo0e2MHpki4PDrI7kPfLYALylTPs7gdVDV46Zme0O8obHT4GL0rE9AELSROAy4LaKVGZmZnUrb3icB+wHvACMAn4FLAdeBb5amdLMzKxe5X0w4ivp49iPAd5HEjoPAIsjwoNEmZUxMO76zacfUeNKzIZervCQNCUingLuSL+y8z4cEfdUoDYzM6tTeU9b/VLSn5Q2SvoLkushZmbWRPKGx53AHZIGbg5E0lFAB3B+JQozM7P6lTc8Pgf8J/AzSSMzwfGViJhfserMzKwu5QqP9KL4icB64C7gJ8CXIuKqCtZmZmZ1arsXzCW9t0zzN4AfAv8M/GZgmYj4XWXKM3NvJbN6tKPeVg8BASjTNjB9BskDEpW2Dc/zZpJmAlemyy+IiMtK5p8LnAb0k9y5/tmIWJHOewN4OF30mYiYlec9zcxs6O0oPKYN5RtJGg7MJ7lXpBdYIqkjIpZlFnsQaIuIVyV9DrgcOD6dtz4iDh3KmszMbOdsNzwi4skhfq/DgOUR0QMgaSEwG9gcHhFxd2b5TuCkIa7BzMyGQN6n6g4cOcwAJgIjsvMi4qYcmxgHrMxM9wKH72D5U4FfZKZHSuoiOaV1WUT8OE/dZmY29PLeYX4gyc2A70ibgqSn1hvARiBPeKhMW9lHm0g6CWgD/jzTPDEiVkmaCvyrpIdLj44kzQXmAkycODFHSWZmtjPy3ufxHeB3wBiShyEeDLSTXKP4RM5t9AITMtPjScYD2Yqko0ketjgrIjYPPBURq9LvPcA9wPTSdSPi2ohoi4i21tbWnGWZmVlRecPjcGBeRKwFNgHDIuJ+4MvAFTm3sQSYJmmKpBHAHJIbDTdLh7u9hiQ4Xsi0j5G0Z/p6X+ADZK6VmNWjdRs28uzL6+le0VfrUsyGXN7wGAb8MX39IrB/+nolOXtlRUQ/cBawGHgUuCUilkqaJ2mg2+23gDcDP5L0kKSBcDkY6JL0H8DdJNc8HB5Wt7pX9PHY8+vo7VvPiQs6HSDWcPJeMH8EeC/QA9wHfFnS6yTXF3L3yoqIRcCikraLMq+P3s56vwXek/d9zGqts2cNm9Irehv7N9HZs8bD6FpDyRselwJ7pa8vIgmAfwdeAv5rBeoy2621Tx3LMMGmgJY9htE+dWytSzIbUnkHg/pF5vVy4MD0Ee0vRsSmShVn9cGDGhU3Y9IYDtpvNGs39HPlnOk+6rCGs8NrHpLeK6nsMhHxgoPDbPtGj2xh3N6jHBzWkAa7YP4gsO/AhKSfS3p7ZUsyM7N6N1h4lN7YdyQwqkK1mJnZbiJvV13bDRx/zb2br0+YmVXSYOERbPsIkbKPFDEzs+YxWG8rAf8iaeAxISOB70t6NbuQx9YwM2sug4XHDSXT/1KpQszMbPexw/CIiFOqVYjVr3UbNrJ2Qz/dK/rc7dTMAF8wt0H4GU1mVo7Do4FU4imu5Z7RZGbm8GgQlTpCGHhGE/gZTWa2hcOjQVTqCGHgGU3jx4zixtPafc3DzIACY5hbfT8gsJJPcR09soXRI1scHAXV48+J2VBxeDQIP8XVzKrJp60KqPdhRf0UVzOrFodHTu6yama2hcMjJ3dZNTPbwuGRk7usmplt4QvmOe0OF6Qr1bvHvYbMrJTDo4BF5xxZ6xLMzOqCT1uZmVlhDg8zMyvM4WFmFeOhkRuXw8PMzApzeJiZWWFVDQ9JMyU9Lmm5pPPLzD9X0jJJv5N0l6RJmXknS3oi/Tq5mnWbmdnWqhYekoYD84FjgUOAEyQdUrLYg0BbRLwXuBW4PF13H+DrwOHAYcDXJdXfjRZmZk2imkcehwHLI6InIl4HFgKzswtExN0R8Wo62QmMT19/DPhlRLwUEX3AL4GZVarbzMxKVDM8xgErM9O9adv2nAr8YifXNTOzCqrmHeYq0xZlF5ROAtqAPy+yrqS5wFyAiRMn7lyVZmY2qGoeefQCEzLT44FVpQtJOhr4KjArIl4rsm5EXBsRbRHR1traOmSFm5ntLqp1b001w2MJME3SFEkjgDlAR3YBSdOBa0iC44XMrMXARyWNSS+UfzRtMzOzGqjaaauI6Jd0Fskf/eHA9RGxVNI8oCsiOoBvAW8GfiQJ4JmImBURL0m6mCSAAOZFxEvVqt3MzLZW1afqRsQiYFFJ20WZ10fvYN3rgesrV52ZmeXlO8zNzKwwh0eF+cFwZtaIHB5mZlaYw8PMKmbdho08+/J6ulf01boUG2IOjwrzL481q+4VfTz2/Dp6+9Zz4oJO/w40GIdHBfmXx5pZZ88aNqXPgdjYv4nOnjW1LciGlMOjgvzLY82sfepYhqUPFmrZYxjtU8fWtiAbUlW9z6PZDPzybAr/8ljzmTFpDAftN5q1G/q5cs50ZkzyKAqNxOFRQf7lsWY3emQLo0e2+Ge/ATk8dtHAPRw3n35E2fn+5TGzRuRrHrtoKHtT+YZCM9td+MhjFwz0ptoUcOKCTm48rX2bI4ztHZGYme3OfOSxC9ybysyalcNjF7gropk1K4fHLhjoTTV+zKiyp6yK8t3oZra7cHjsotEjWxi396hdDg7fjW5mQ6Fa/4Q6PHbRzacfMSQXxX39xMx2VTX/CXV41AlfPzGzXVXNf0LdVbdO+G50M9tV1XwkksOjjvhudDPbFdX8J9ThYWbWQKr1T6jDo474bnRrNP6Zbly+YG5mZoU5PMzMrDCHh5mZFebwMDOzwhweZmZWmMPDzMwKq2p4SJop6XFJyyWdX2b+kZIekNQv6VMl896Q9FD61VG9qs3MrFTV7vOQNByYDxwD9AJLJHVExLLMYs8AnwG+WGYT6yPi0IoXamZmg6rmTYKHAcsjogdA0kJgNrA5PCLi6XTepirWZWZmBVXztNU4YGVmujdty2ukpC5JnZKOK7eApLnpMl2rV6/elVrNzGwHqhkeKtMWBdafGBFtwF8B35F0wDYbi7g2Itoioq21tXVn6zQzs0FUMzx6gQmZ6fHAqrwrR8Sq9HsPcA8wfSiLMzOz/KoZHkuAaZKmSBoBzAFy9ZqSNEbSnunrfYEPkLlWYmZm1VW18IiIfuAsYDHwKHBLRCyVNE/SLABJfyapF/g0cI2kpenqBwNdkv4DuBu4rKSXlpmZVVFVH8keEYuARSVtF2VeLyE5nVW63m+B91S8QDMzy8V3mJuZWWEeDMrMrIFUawAuH3mYmVlhDg8zMyvM4WFmZoU5PMzMrDCHh5mZFebwMDOzwhweZmZWmMPDzMwKc3iYmVlhiigypMbuQ9JqYMUgi+0LvFiFcupRs352f+7m4s9d3KSIGHRApIYNjzwkdaUDTDWdZv3s/tzNxZ+7cnzayszMCnN4mJlZYc0eHtfWuoAaatbP7s/dXPy5K6Spr3mYmdnOafYjDzMz2wlNGx6SZkp6XNJySefXup5KkTRB0t2SHpW0VNI5afs+kn4p6Yn0+5ha11oJkoZLelDSz9LpKZLuSz/3zZJG1LrGoSZpb0m3Snos3e9HNMP+lvSF9Gf8EUk/lDSyUfe3pOslvSDpkUxb2X2sxD+kf+t+J+l9Q1FDU4aHpOHAfOBY4BDgBEmH1LaqiukHzouIg4F24Mz0s54P3BUR04C70ulGdA7waGb6fwNXpJ+7Dzi1JlVV1pXA7RFxEPCnJJ+/ofe3pHHA2UBbRLwbGA7MoXH39z8CM0vatrePjwWmpV9zge8NRQFNGR7AYcDyiOiJiNeBhcDsGtdUERHxXEQ8kL5eR/KHZBzJ570hXewG4LjaVFg5ksYDHwcWpNMCjgJuTRdpuM8t6S3AkcB1ABHxekS8TBPsb5JhtUdJ2gN4E/AcDbq/I+LfgJdKmre3j2cD/xSJTmBvSW/f1RqaNTzGASsz071pW0OTNBmYDtwHvC0inoMkYIA/qV1lFfMd4MvApnR6LPByRPSn042436cCq4EfpKfrFkjaiwbf3xHxLPB3wDMkofEK0E3j7++s7e3jivy9a9bwUJm2hu52JunNwP8FPh8Ra2tdT6VJ+gTwQkR0Z5vLLNpo+30P4H3A9yJiOvBHGuwUVTnp+f3ZwBRgf2AvktM1pRptf+dRkZ/7Zg2PXmBCZno8sKpGtVScpBaS4LgxIm5Lm38/cOiafn+hVvVVyAeAWZKeJjkteRTJkcje6WkNaMz93gv0RsR96fStJGHS6Pv7aOCpiFgdERuB24D30/j7O2t7+7gif++aNTyWANPSnhgjSC6sddS4popIz/NfBzwaEd/OzOoATk5fnwz8pNq1VVJEXBAR4yNiMsn+/deIOBG4G/hUulgjfu7ngZWS3pk2fQRYRoPvb5LTVe2S3pT+zA987obe3yW2t487gP+e9rpqB14ZOL21K5r2JkFJf0nyn+hw4PqIuKTGJVWEpA8C/w48zJZz/39Lct3jFmAiyS/epyOi9AJcQ5D0YeCLEfEJSVNJjkT2AR4EToqI12pZ31CTdChJJ4ERQA9wCsk/ig29vyV9AziepIfhg8BpJOf2G25/S/oh8GGSp+f+Hvg68GPK7OM0TP8PSe+sV4FTIqJrl2to1vAwM7Od16ynrczMbBc4PMzMrDCHh5mZFebwMDOzwhweZmZWmMPDzMwKc3iYmVlhDg+znSRpjKTfSzqg1rUUkY71cW6t67Ddm8PDbOf9LbAoIp7MNkp6m6Qr0kF5NqSD9vxW0t+kD6gclKSfSrpzO/MOlhSSjilpv1rSFTk2/w3ga5LemqcWs3IcHmY7QdKbSB5/cV1J+2TgAZJHQVxI8lDCo0geF/4RYFbOt1gAHJVur9SpwAqSAX8G3lfAJ8nx7KaIeJjksSUn5azFbBsOD2saku5J/zu/UlJf+vUtSTvze/CXJM8K+01J+/fS9raIWBgRyyLikYi4LSKOA36YqUeSvizpSUnrJT0saeAP+s9Jnll0SslnaAH+G8nz2DZlZv0ZMBL4dWbb56VHP69J6pX0zczyHcAJO/G5zQCHhzWfE0l+7o8ATicZlvPzO7GdDwHdkXk4nKR9gI8B8yPij+VWiq0fJve/SI4iziQZDvmbwDWSPp4OYHQD8JmScPskycPwflCy6eOAn2cGPrqU5Mjnm8C7gE+z9YBA9wOHSRqV/yObbeHwsGbzHHB2RDwWEbcA3wLOBZB0hqSH0iOAjenrhyR9ocx2JqXbyppGMvDO49nG9L/+P6RfV6dte6Xve1pE3B4RT0XETcD3ScIEklNiE0nGqhhwKnBHRGSDAJKBkH6cbvvNwBeA8yPi+ohYHhH3RsRVmeVXAS0kAyeZFbbH4IuYNZTOkv/+7wUulvSWiLgauFrSu0jGfJ6xg+2MIjmtlMeHSB79fy3JqSVIjjRGArdLytbTAjwNEBFPSPo34LPAHZL2JzmyOT67cUnvIBl+dnFm23uSuSZSxvrM5zArzOFhtq13A48MssyLwJiStuUkw3selG2MiKcAJL2aaR446v8kydgLWRszrxcA309PiX0GeIltBy47Drgrc6qs3LCjpfZJv6/OsazZNnzayprN4WnPpAHtwKqScd3zhMeDJP/hbxYRa4A7gLNydMldBrwGTEpPK2W/VmSWuxXYQNIz6rMkR0QbS7a1+ZRVybY/soP3fzfJ58579GS2FYeHNZv9ge9IeqekTwFfAkrvjcgTHouBgyWNLWn/a5Lfq25JJ0g6RNKBkk4A/hR4AyAi1pF03/07SZ+V9A5Jh6bXXeYObCwi1gM3Af8TOIBtuwa3kgTgTzPrrAOuBL4p6RRJB0g6TNLnMqt+CLh9kM9otl0+bWXN5kaS6w/3kZxiuo6dCI+IeFjS/STjo8/PtPdImg5cAFwMTCA5DfUocBXJcKADLiS5bvJFki6+a4GHgMtL3m4B8DngtxHxaMm8TwJLyhxBXAD0pe8xPn2ffwKQNBL4LyTXT8x2ioehtaYh6R7gkYg4awfLjAKejYh9trdMZtmZJP/hHxIRbwxZoQVI+gnwm4goDZwdrXMmMDsiPlq5yqzR+bSV2dYOJjlKGFRE3E5y1DG+ohXt2G/I3HiY00bgbypQizURH3lY08hz5GFm+Tg8zMysMJ+2MjOzwhweZmZWmMPDzMwKc3iYmVlhDg8zMyvM4WFmZoU5PMzMrDCHh5mZFfb/Ab+CSdnax2bHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create efficiency/fake rate vs. pt\n",
    "pt, eff, faker, err_eff, err_faker = get_eff_faker_vs_feat('pt',features,X_test,y_test,clf_GBDT)\n",
    "\n",
    "plt.errorbar(pt,eff,yerr=err_eff,linestyle='None',fmt='.')\n",
    "plt.xlabel('p$_{T}$ (GeV/c)',fontsize=14)\n",
    "plt.ylabel('Efficiency',fontsize=14)\n",
    "plt.show()\n",
    "\n",
    "plt.errorbar(pt,faker,yerr=err_faker,linestyle='None',fmt='.')\n",
    "plt.xlabel('p$_{T}$ (GeV/c)',fontsize=14)\n",
    "plt.ylabel('Fake rate',fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEPCAYAAAC3NDh4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFAFJREFUeJzt3X+QXWd93/H3R5KNZ4qLjSSGRJJ/NYLiZsCOt0Yp0zYEnJGZjj1kcGvzIySYuJNiUkqgNXUh1PmjSZiBlNaJozoeSHBwDW2NGpwYmjplpkaMV8Z1sY0bRamsjUkthEpIgchC3/6xV2FZ3Ud7z/ree3ZX79eMRvec89xzv3fv7vnc5zznR6oKSZKGWdd3AZKklcuQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKlpQ98FPFubNm2qCy64oO8yJGlV2bt371eravNS7VZ9SFxwwQXMzs72XYYkrSpJDozSzt1NkqQmQ0KS1GRISJKaphYSSe5I8nSSLzWWJ8mHk+xL8kiSH5pWbZKk4abZk/gIsPMUy68Etg/+3QD82hRqkiSdwtRCoqo+B3ztFE2uBn6z5u0BzknyfdOpTpI0zEoak9gCHFwwPTeY17u9B45w6/372HvgSN+lSNJUraTzJDJk3tB7qya5gfldUpx33nmTrIm9B47whtv3cPTYcc7csI4737qDy84/t9l2z/7D7LhoY7PNctpK0mLT2oaspJCYA7YtmN4KPDWsYVXtAnYBzMzMTPQm3Xv2H+boseMcL3jm2HH27D889APpGiajtpV0+hh1wz/NbchK2t20G/iJwVFOO4CvV9VX+i5qx0UbOXPDOtYHztiwjh0XbRzabliYtHRpuxZNYvfdqOvs8tp97mbsu053sY7XKD/PvQeOcM1tD/CB+57gDbfvOWXbaW5DptaTSPJx4EeATUnmgJ8HzgCoqtuAe4HXAPuAbwI/Na3aTuWy88/lzrfuWDLdT4TJM8eOnzJMurZda078IRwvOOuM8ey+G/Vb1aR6e+Pezdh3r7TvXayT2I3S5zpH/Xku3NCfaq8FTHcbMrWQqKrrllhewNumVE4nl51/7pK/WKOGSde2XfQ5zjHqa4/6h9BlQ7Vn/2G+/czxJdc5arsTbfvazTjqa3dt2+UzGuXnNIn33jX0Rg3dUb+YdFlnl9/PUT6jLhv+SW1DhllJYxKr3ihh0rXtJPZR9vnNd9Q/hC4bvx0XbeSsM5Ze56jtJlXnJDYWo7bt+hmN8nOaxHvvEuST+IY+iSAf9TPquuHvsr15NgyJFWwS36b7/uY7id13o65zEr29SWzQJ1Fn35/RqG27BPkkQrfvb/3T2vB3YUisYJP4Nt33N18Y/+67UdfZpd0k6pzUxmKUtn1/RpMI8kmE7mr/1j8JmR8KWL1mZmZqrd5P4sQ3+RO/sOPclzrKOru29byPlW0tfkarZZB7JUqyt6pmlmxnSKxsff8RnC5/MNLpxpCQJDWNGhIr6WQ6SdIKY0hIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUNNWQSLIzyRNJ9iW5acjy85Lcn+SLSR5J8ppp1idJ+l5TC4kk64FbgSuBi4Hrkly8qNm/AO6uqkuBa4FfnVZ9kqSTTbMncTmwr6r2V9VR4C7g6kVtCvirg8fPA56aYn2SpEU2TPG1tgAHF0zPAS9f1Ob9wGeSvB34K8Crp1OaJGmYafYkMmReLZq+DvhIVW0FXgP8VpKTakxyQ5LZJLOHDh2aQKmSJJhuSMwB2xZMb+Xk3UnXA3cDVNXngbOATYtXVFW7qmqmqmY2b948oXIlSdMMiQeB7UkuTHIm8wPTuxe1eRJ4FUCSlzAfEnYVJKknUwuJqjoG3AjcBzzO/FFMjya5JclVg2Y/B/x0kv8BfBz4yapavEtKkjQl0xy4pqruBe5dNO99Cx4/BrximjVJkto841qS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqSmTiGR5JJJFSJJWnm69iQeSrI3yc8ked5EKpIkrRhdQ+LFwGeAm4GvJPlYkleOvyxJ0krQKSSq6g+r6j3AecA1wFnA7yX5oyQ3J9k6iSIlSf1Y1sB1VR2vqk8DbwRuArYAvwDsT3JXki1jrFGS1JNlhUSSy5PcBnwFeAfwi8AFwN8Gng/cM64CJUn92dClcZJ3Am8BtgOfBt4A/F5VHR80eTLJPwK+PNYqJUm96NqT+BngY8B5VfXjVXXvgoA44Ung+mFPTrIzyRNJ9iW5qdHm7yd5LMmjSX67Y32SpDHq1JOoqu0jtDkKfHTx/CTrgVuBK4A54MEku6vqsQVttgPvAV5RVUeSvKBLfZKk8ep6Mt2NSd44ZP4bB7uZTuVyYF9V7R8EyV3A1Yva/DRwa1UdAaiqp7vUJ0kar667m94BHBwy/38D/2SJ525Z9Ny5wbyFXgS8KMl/T7Inyc5hK0pyQ5LZJLOHDh0arXJJUmddQ2IrcGDI/LnBslPJkHm1aHoD84PiPwJcB9ye5JyTnlS1q6pmqmpm8+bNSxYtSVqeriHxp8Cw6zf9EPDVJZ47B2xbML0VeGpIm09V1TNV9cfAE8yHhiSpB11D4reBDye5IskZg38/BvwKcOcSz30Q2J7kwiRnAtcCuxe1uQd4JUCSTczvftrfsUZJ0ph0OroJ+HngQuA+4DuDeeuATwDvPdUTq+pYkhsHz10P3FFVjya5BZitqt2DZT+W5LHB+t9dVYc71ihJGpNULR4WGOFJ84eqXjqYfKiq9o21qg5mZmZqdna2r5eXpFUpyd6qmlmqXdeeBDB/oT/gD5fzXEnS6tE5JJL8A+BVwAtYNKZRVVeNqS5J0grQ9dpNH2D+XIn7mT8yqfu+KknSqtG1J/ETwHVV9clJFCNJWlm6HgK7Dnh4EoVIklaeriGxi/kbDUmSTgNddzedA7w+yRXAI8AzCxdW1c+OqzBJUv+6hsTFfHd3019ftMxBbElaY7reT+KVkypEkrTyLPce15uSvDzJc8ZdkCRp5eh606Gzk3wCeBp4gMH9IJLcluT94y9PktSnrj2JXwK+n/lLg39rwfzfAV47rqIkSStD14Hrq4DXVtXDSRYOVD8OXDS+siRJK0HXnsS5wLBLd5/Ndy8dLklaI7qGxIPM9yZOONGb+IfMj1FIktaQrrub/jlwX5K/MXjuOwePLwf+zriLkyT1q1NPoqoeAP4WcCbwR8xfMvwp4Ier6qHxlydJ6lPn+0lU1f8E3jyBWiRJK8ySIZHk+VX1tROPT9X2RDtJ0towSk/iUJLvq6qnga8y/BpNGcxfP87iJEn9GiUkfhQ40UPw2k2SdBpZMiSq6r8NeyxJWvu6XrvpmiRXD5l/dZLXja8sSdJK0PVkuvcD3x4y//8NlkmS1pCuIXER8MSQ+fvw2k2StOZ0DYkjwPYh818EfOPZlyNJWkm6hsSngA8ledGJGUleDHwQuGechUmS+tc1JP4p8HXgsSQHkxwEHgX+DHj3uIuTJPWr6z2uvwG8IskVwCXMn0T3EPD7VTXsJDtJ0irW+dpNAFX1WeCzY65FkrTCjHLtpncCv1pV3x48bqqqD46tMklS70bpSbwd+Cjz50e8/RTtivkB7KYkO4F/zfw1nm6vql9stHsd8Angb1bV7Ag1SpImYJTLclw47HFXSdYDtwJXAHPAg0l2V9Vji9qdDfws8IXlvpYkaTyWPLopyXeSvGDw+I7BRnw5Lgf2VdX+qjoK3AWcdIkP4BeAX2b4md2SpCka5RDYbwHPHTx+M3DWMl9rC3BwwfTcYN5fSnIpsK2qfudUK0pyQ5LZJLOHDh1aZjmSpKWMMibxAHBPkr3MH/L64STfGtawqt5yivVk2FP+cmGyDvgQ8JNLFVRVu4BdADMzMx56K0kTMkpIvAl4F/ADg+mNwF8s47XmgG0Lprcyf3/sE84GfhD4gyQALwR2J7nKwWtJ6scoA9f/h8HZ1En+GLiuqg4v47UeBLYnuRD4E+Ba4PULXufrwKYT00n+AHiXASFJ/ek0cA3cDxxdzgtV1THgRuA+4HHg7qp6NMktSa5azjolSZOVpa6mkeTPgZdW1f4k3wFeWFUrZrR4ZmamZmftbEhSF0n2VtXMUu2mOXAtSVplRh24fjfw15g/Gmm5A9eSpFVm1IHrd8GzHriWJK0yI91PIskDSc6pqgur6nCSf5Xk+QuWb0ry5OTKlCT1YdSbDu0Azlww/TbgnAXT65k/70GStIZ0vTPdCac8e1qStDYsNyQkSaeBUUOiOLmnYM9Bkta4UW9fGuBjSU4c+noW8O+SfHMw/ZyxVyZJ6t2oIfHRRdMfG9LmN59lLZKkFWakkKiqn5p0IZKklceBa0lSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElNUw2JJDuTPJFkX5Kbhix/Z5LHkjyS5PeTnD/N+iRJ32tqIZFkPXArcCVwMXBdkosXNfsiMFNVLwU+CfzytOqTJJ1smj2Jy4F9VbW/qo4CdwFXL2xQVfdX1TcHk3uArVOsT5K0yDRDYgtwcMH03GBey/XA7060IknSKW2Y4mtlyLwa2jB5IzAD/N3G8huAGwDOO++8cdUnSVpkmj2JOWDbgumtwFOLGyV5NXAzcFVV/cWwFVXVrqqaqaqZzZs3T6RYSdJ0Q+JBYHuSC5OcCVwL7F7YIMmlwK8zHxBPT7E2SdIQUwuJqjoG3AjcBzwO3F1Vjya5JclVg2YfAJ4LfCLJw0l2N1YnSZqCaY5JUFX3Avcumve+BY9fPc16JEmn5hnXkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpaaohkWRnkieS7Ety05Dlz0ny7wfLv5DkgmnWJ0n6XlMLiSTrgVuBK4GLgeuSXLyo2fXAkar6AeBDwC9Nqz5J0smm2ZO4HNhXVfur6ihwF3D1ojZXAx8dPP4k8KokmUQxew8c4db797H3wJFJrF6S1oQNU3ytLcDBBdNzwMtbbarqWJKvAxuBr46zkL0HjvCG2/dw9NhxztywjjvfuoPLzj93nC8hSWvCNHsSw3oEtYw2JLkhyWyS2UOHDnUuZM/+wxw9dpzjBc8cO86e/Yc7r0OSTgfTDIk5YNuC6a3AU602STYAzwO+tnhFVbWrqmaqambz5s2dC9lx0UbO3LCO9YEzNqxjx0UbO69Dkk4H09zd9CCwPcmFwJ8A1wKvX9RmN/Bm4PPA64D/WlUn9SSercvOP5c737qDPfsPs+Oije5qkqSGqYXEYIzhRuA+YD1wR1U9muQWYLaqdgO/AfxWkn3M9yCunVQ9l51/ruEgSUuYZk+CqroXuHfRvPctePxt4Jpp1iRJavOMa0lSkyEhSWoyJCRJTYaEJKnJkJAkNWUCpyFMVZJDwIG+61iGTYz5ciOrwOn2nk+39wu+59Xk/Kpa8mzkVR8Sq1WS2aqa6buOaTrd3vPp9n7B97wWubtJktRkSEiSmgyJ/uzqu4AenG7v+XR7v+B7XnMck5AkNdmTkCQ1GRI9SfKBJF9O8kiS/5TknL5rmrQk1yR5NMnxJGv2aBCAJDuTPJFkX5Kb+q5n0pLckeTpJF/qu5ZpSbItyf1JHh/8Xv/jvmuaBEOiP58FfrCqXgr8L+A9PdczDV8Cfhz4XN+FTFKS9cCtwJXAxcB1SS7ut6qJ+wiws+8ipuwY8HNV9RJgB/C2tfg5GxI9qarPVNWxweQe5u/Ut6ZV1eNV9UTfdUzB5cC+qtpfVUeBu4Cre65poqrqcwy5i+RaVlVfqaqHBo+/ATwObOm3qvEzJFaGtwC/23cRGpstwMEF03OswY2HvivJBcClwBf6rWT8pnrTodNNkv8CvHDIopur6lODNjcz3229c5q1Tcoo7/k0kCHzPIxwjUryXOA/AO+oqj/ru55xMyQmqKpefarlSd4M/D3gVZO4l3cflnrPp4k5YNuC6a3AUz3VoglKcgbzAXFnVf3HvuuZBHc39STJTuCfAVdV1Tf7rkdj9SCwPcmFSc5k/l7tu3uuSWOWJMBvAI9X1Qf7rmdSDIn+/FvgbOCzSR5OclvfBU1aktcmmQN+GPh0kvv6rmkSBgck3Ajcx/xg5t1V9Wi/VU1Wko8DnwdenGQuyfV91zQFrwDeBPzo4G/44SSv6buocfOMa0lSkz0JSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIYzY4qepfJvk3Sb46uM/Ce/uuS1oOT6aTxmhwLZ8/Z/76Te9n/izkNwHvBb6/qv60v+qk7gwJaYySvAx4GHhtVd0zmLcNeBJ4SVV9uc/6pK7c3SSN1yXAEeA/L5i3efC/vQitOoaENF4vA2ar6jsL5l0CHKiq/9tTTdKyGRLSeF0CfHHRvEuZ3wUlrTqGhDReL2V4SCyeJ60KhoQ0Jkm2AhtZ0GsY3JjmZRgSWqW8fak0JlU1x8n3t94OPBdDQquUPQlpsi4FDlfVwb4LkZbDkJAmy0FrrWqeTCdJarInIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVLT/wfDzdYY2Nf1twAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEPCAYAAAC3NDh4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFkhJREFUeJzt3X/wXXdd5/HnO2liOkuEThopJmmTYmDJaGkn321T/I04m65O6g+YpqAD2tq6a0YYGaVMaWapswyLo8iyYWgsKIPFFBE1YLAFF0WcBvON7bCEEviSJeZLdfi2xv4QQvo17/3j3i/cfnPP93u+yT3n3HPv8zGTyb3nnJ7zPm3vfd3zOZ/P50RmIklSP8uaLkCSNLwMCUlSIUNCklTIkJAkFTIkJEmFDAlJUiFDQpJUyJCQJBUyJCRJhS5ouoDzdfHFF+fGjRubLkOSWuXw4cOPZubaxbZrfUhs3LiRycnJpsuQpFaJiONltrO5SZJUqNaQiIjtEXE0IqYi4rY+618TETMR8VD3z8111idJeqbampsiYjmwB/hxYBo4FBH7M/Pz8za9NzN31VWXJKlYnVcSVwNTmXksM08D+4Drazy+JGmJ6gyJdcCJnvfT3WXz/WxEfDYiPhQRG+opTZLUT50hEX2WzX/i0UeAjZl5BfAJ4H19dxRxS0RMRsTkzMzMgMuUJM2pMySmgd4rg/XAI70bZOZjmfnN7tvfA7b221Fm7s3MicycWLt20W6+kqRzVGdIHAI2R8SmiFgJ7AT2924QEc/rebsDeLjG+iRJ89TWuykzZyNiF3AfsBx4b2YeiYg7gcnM3A/8akTsAGaBfwFeU1d9g3DDXQ8AcO+t1zZciSQNRq0jrjPzAHBg3rLdPa/fCLyxzpokScUccS1JKmRISJIKGRKSpEKGhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhMUBPnnqar/7rNzh8/GTTpUjSQBgSA3L4+Em+8M9PMn3yG7zq7oMGhaSRYEgMyMFjj3GmO/H507NnOHjssWYLkqQBMCQGZNvla1jWfWLGiguWse3yNc0WJEkDUOsEf6Ns62UX8R8vWc0Tp2Z5x86r2HrZRU2XJEnnzZAYoNWrVrB61QoDQtLIsLlJklTIkJAkFTIkJEmFDAlJUiFDQpJUyJCQJBUyJBpyw10PcMNdDzRdhiQtyJCQJBVyMN0A3XvrtU2XIEkD5ZWEJKmQISFJKmRISJIKGRIN8Sl2ktrAkGiAT7GT1BaGRAN8ip2ktjAkGuBT7CS1heMkGuBT7CS1Ra1XEhGxPSKORsRURNy2wHYvj4iMiIk666vT6lUrWPecCw0ISUOttpCIiOXAHuA6YAtwY0Rs6bPdauBXgc/UVZskqb86rySuBqYy81hmngb2Adf32e43gbcBp2qsTZLUR50hsQ440fN+urvsWyLiKmBDZn60xrokSQXqDInosyy/tTJiGfB24PWL7ijiloiYjIjJmZmZAZbYn9N6SxpXdfZumgY29LxfDzzS83418L3AX0cEwCXA/ojYkZmTvTvKzL3AXoCJiYmkhZwxVlIb1HklcQjYHBGbImIlsBPYP7cyMx/PzIszc2NmbgQOAmcFhCSpPrWFRGbOAruA+4CHgQ9m5pGIuDMidtRVhySpvFoH02XmAeDAvGW7C7b9kTpqkiQVc1oOSVIhQ0KSVMiQkCQVMiQkSYUMCUmqWJsH5BoSJfioUUnjypBYhI8alTTODIlF+KhRSePMkFiEjxqVNM58fOkifNSopHHmlUQJPmpU0vloc+cXQ0KSKtT2zi+GhCRVqO2dXwwJSapQ2zu/eONakirU9s4vhoQkVWz1qhWsXrWidQEBNjdJkhZgSEiSChkSkqRC3pMo4d5br226BEl6hrmpx6v+fvJKQpLOQZufEbEUXklIUsXa3BrhlYQkqZAhIUkqZEhI0jlo88yuS2FISNIStX1m16UoHRIRsTYiXhcR74yINd1l2yLisurKk6Th0/aZXZeiVEhExFXAUeAm4Fbg2d1V1wFvqaY0SRpObZ/ZdSnKXkn8NvCuzPw+4Js9y/8S+IGBVyVJDSkz/mFuZtf1F13IPTdva+XEfWWVHSexFfilPssfAZ47uHIkqR3aPLPrUpS9kjgFfGef5S8EZgZXjiRpmJQNiY8AuyNiRfd9RsSlwFuBD5c9WERsj4ijETEVEbf1Wf/LEfF/I+KhiPh0RGwpu29JGid1dcEtGxKvBy4BvgZcCPwNMAV8Hbi9zA4iYjmwh87N7i3AjX1C4AOZ+X2ZeSXwNuB3StYnSQNR9sv33luvbWy6jTq74JYKicx8HHgJcAPwJuDdwA7gBzPzqZLHuhqYysxjmXka2AdcP+84T/S8/Q9Alty3JJ23tox/qLMLbqkb1xHxSuBDmXk/cH/P8pUR8fLM/ECJ3awDTvS8nwau6XOsXwF+DVgJvLRMfZI0CP2+fIfxxvRcF9wzWX0X3LLNTe8HntNn+eruujKiz7KzrhQyc09mPh94A52rlrN3FHFLRExGxOTMjPfNJQ1GW8Y/1NkFt2xIBP2bftYDT/RZ3s80sGHeP/vIAtvvA36q34rM3JuZE5k5sXbt2pKHl6SFtWn8w+pVK1j3nAsrr3HB5qaIeJBOOCTwVxEx27N6ObCJnuanRRwCNkfEJuCrwE7glfOOtzkzv9R9+xPAlxhzdT19SlLHuIx/KGuxexIf7f59JfAJ4N961p0GvgL8cZkDZeZsROwC7qMTMO/NzCMRcScwmZn7gV0R8TLgaeAk8OqyJyJp/PgjqnoLhkRm3gEQEV8B7snMU+dzsMw8AByYt2x3z+vXns/+JUmDVap3U2a+p+pC1N+Tp57miVOzHD5+0stfqQZelTxT2VlgV0TEHRHx+Yh4KiJO9/6pushx1ZY+21JTxuXBP00q27vpTjoT/O2hcz/hduBu4HHAJqKKjNOc9dJS+SOqHmVD4gbg1szcA8wCH87M/wa8GfjRqoobd23psy01wR9R9SgbEpcAR7qvn+LbA+sOAP950EWpo019tqW6+SOqHmVD4gTwvO7rLwM/3n19NZ1pxFWRugbMSG3jj6h6lA2J/Xw7GN4J/GZEfAl4H/D7VRQmSYvxR1T1ynaB/fWe1/dGxFfpzAr7xcz8s6qKk6SFjHN31brOfdGQ6D5o6A+A3Zn5ZYDM/DTw6WpLE4z3h0BS8xZtbsrMp+nMo+SzHVSozMPjJbVP2XsSf0rBjKySpNFV6p4EnR5Nd0TES4DDPHOiPzLzfw26MLWL04doUJy0b7iUDYlb6YyPuIaznyaXgCExBJr6cM2NfD2T8Kq7D9odURohZXs3bVh8K42rtjzyUdLSlb0nIRWqauSrN8PHk5P2DRdDYoQ09eFa6shXv/xVxEn7ho8hMSKa/nA58lWD4KR9w8eQGBF+uDQKnLRv+JTt3aQhN/fhOpPNfLiq6FFlt9rxM9d0+cSpWd6x8yr/uw+B0lcSEbE2Il4XEe+MiDXdZdsi4rLqylNZbZoRs8y9k6abz9Qcmy6HS9nHl14FHAVuojNm4tndVdcBb6mmNC1VGz5cZb/8bT6ThkPZ5qbfBt6VmW+KiCd7lv8lsG/wZWlUlR1T0XTzmcqpYgCnI62HS9mQ2ErnGdfzPQI8d3Dl6Hy04cNV9svftmkNklN9nLuyIXEK+M4+y18IzAyuHI26pXz5r161gtWrVhgQUoPK3rj+CLC7+2wJgIyIS4G3Ah+upDKNrDbcO1E5jo4efWVD4vXAJcDXgAuBvwGmgK8Dt1dTmsbdvbdea/PAEGtTDzTD7NyVConMfJzO40pvAN4EvBvYAfxgZj5VXXmShlVbeqC1KcyGUal7EhGxKTP/H3B/90/vuh/JzL+uoDYNAW/4qUhbeqA5S/H5KXvj+uMR8ZLM/Frvwoj4UWA/sHrglWlkNRk4ht7gtKUHWlvCbFiVvSfxCeD+iJgbREdEvJROQNxWRWEaDrblaiFt6ITQptkIhlHZkPivwBeBj0bEqp6AeENm7qmsOjXKttzBc5r0ZrQhzIZV2RvXCbwK+AbwV8CfA7+eme+qsDY1rC03JpfCKyNpaQrvSUTEFX0Wvxn4I+D9wN/NbZOZny1zsIjYDrwDWA7cnZlvnbf+14CbgVk6g/R+MTOPl9m3Bm/U2nKH4VncozazbVvu7bSlzmG8Z7bQlcRDwIPdv+f+fApYD/xyz7oHyxwoIpYDe+hMCrgFuDEitszb7EFgIjOvAD4EvK30mWjgRq0tt+kro3FvvrOprZ0W6t20ecDHuhqYysxjABGxD7ge+PzcBpn5yZ7tDwI/N+AatESjNDVG01dGdsVUGxWGRGZ+ecDHWgec6Hk/DVyzwPY3AR/rtyIibgFuAbj00ksHVZ/6GKbL3vPVdJfNpkNqGJsyNPxKP5mu21y0FbgUWNm7LjM/UGYXfZZlwbF+DpgAfrjf+szcC+wFmJiY6LsPqZ8mr4yaDinpXJQdcf0COpP8fU93UdK5n/HvwNNAmZCYBjb0vF9PZ6rx+cd6GZ35oH44M79Zpj4tzTj/omz6nJsMqaZvmjd9fJ2bsuMkfhf4LHARnUn9XgRso3Oj+SdL7uMQsDkiNkXESmAnnbEW39J9At5dwI75o7ululVxo7WKSQvL1LnUm+aDPvdxv2nfZmVD4hrgzsx8AjgDLMvMvwd+A3h7mR1k5iywC7gPeBj4YGYeiYg7I2JHd7PfAp4F/HFEPBQR+wt2p/PgWIHx03TPrqaP3xbD+NksGxLLgH/rvn4U+O7u6xMsoRdUZh7IzBdk5vMz8390l+3OzP3d1y/LzOdm5pXdPzsW3qOWyl905Q3jB7afMnXO3TSHcjfNB33uSz3+OBrWz2bZkPgcMDe47jPAb0TE9wO7gUH3glKF/EVXzlI+sE32/y9b51LGvFTxZTVqY26qMKyfzbIh8Ra+fZN7N/B84G+BnwBeW0Fdqoi/6MoZ1g/sfEups+z8RVWdu/MnLWxYP5ulejdl5sd6Xk8BL4iI7wIezcwzVRWnwRvVbpiD7rG1lDENVfTaKXs+VYy9aHo8x7ga1s/mgiHRnZvpc/2CwN5H7TVKo6irUvYD2/R8UEv5YikboMP6ZTUOhvGzuVhz04PAxXNvIuIvIuJ51ZYkDYcyzSPD0CxVRTOOTUOas1hz0/xR0j8EXFhRLapJ0wPKqlBFk0+Zf09VNc2M4sCzUfz/bhyUnpZDGlZNNvlU0TSz1POp4svXL3TNWay5KTl7fiXnStJQabrJZ9BNM02fj9SrTHPTH0bE3BxKq4Dfi4iv927koDc1qeneOIP+1d30+ag5w3gFt1hIvG/e+z+sqhDpXI1ab5xROx+124IhkZm/UFch0vkYxq6D52PUzkftFZntvsUwMTGRk5OTTZchSa0SEYczc2Kx7cpOyyFJGkOGhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhIUkqZEhIkgoZEpKkQoaEJKmQISFJKmRISJIKGRKSpEKGhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhIUkqVGtIRMT2iDgaEVMRcVuf9T8UEf8QEbMR8fI6a5Mkna22kIiI5cAe4DpgC3BjRGyZt9k/Aq8BPlBXXZKkYhfUeKyrganMPAYQEfuA64HPz22QmV/prjtTY12SpAJ1NjetA070vJ/uLpMkDak6QyL6LMtz2lHELRExGRGTMzMz51mWJKlInSExDWzoeb8eeORcdpSZezNzIjMn1q5dO5DiJElnqzMkDgGbI2JTRKwEdgL7azy+JGmJaguJzJwFdgH3AQ8DH8zMIxFxZ0TsAIiI/xQR08ArgLsi4khd9UmSzlZn7yYy8wBwYN6y3T2vD9FphpIkDQFHXEuSChkSkqRChoQkqZAhIUkqZEhIkgoZEpKkQoaEJKmQISFJKmRISJIKGRKSpEKGhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhIUkqZEhIkgoZEpKkQoaEJKmQISFJKmRISJIKGRKSpEKGhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhIUkqZEhIkgoZEpKkQoaEJKlQrSEREdsj4mhETEXEbX3Wf0dE3Ntd/5mI2FhnfZKkZ6otJCJiObAHuA7YAtwYEVvmbXYTcDIzvwd4O/A/66pPknS2Oq8krgamMvNYZp4G9gHXz9vmeuB93dcfAn4sIqKKYg4fP8meT05x+PjJKnYvSSPhghqPtQ440fN+GrimaJvMnI2Ix4E1wKODLOTw8ZO86u6DnJ49w8oLlnHPzdvYetlFgzyEJI2EOq8k+l0R5DlsQ0TcEhGTETE5MzOz5EIOHnuM07NnOJPw9OwZDh57bMn7kKRxUGdITAMbet6vBx4p2iYiLgCeDfzL/B1l5t7MnMjMibVr1y65kG2Xr2HlBctYHrDigmVsu3zNkvchSeOgzuamQ8DmiNgEfBXYCbxy3jb7gVcDDwAvB/5PZp51JXG+tl52EffcvI2Dxx5j2+VrbGqSpAK1hUT3HsMu4D5gOfDezDwSEXcCk5m5H3gP8P6ImKJzBbGzqnq2XnaR4SBJi6jzSoLMPAAcmLdsd8/rU8Ar6qxJklTMEdeSpEKGhCSpkCEhSSpkSEiSChkSkqRCUcEwhFpFxAxwvOk6zsHFDHi6kRYYt3Met/MFz7lNLsvMRUcjtz4k2ioiJjNzouk66jRu5zxu5wue8yiyuUmSVMiQkCQVMiSas7fpAhowbuc8bucLnvPI8Z6EJKmQVxKSpEKGREMi4rci4gsR8dmI+NOIeE7TNVUtIl4REUci4kxEjGxvEICI2B4RRyNiKiJua7qeqkXEeyPiaxHxuaZrqUtEbIiIT0bEw93/r1/bdE1VMCSa83HgezPzCuCLwBsbrqcOnwN+BvhU04VUKSKWA3uA64AtwI0RsaXZqir3B8D2pouo2Szw+sx8EbAN+JVR/O9sSDQkM+/PzNnu24N0ntQ30jLz4cw82nQdNbgamMrMY5l5GtgHXN9wTZXKzE/R5ymSoywz/ykz/6H7+kngYWBds1UNniExHH4R+FjTRWhg1gEnet5PM4JfHvq2iNgIXAV8ptlKBq/Whw6Nm4j4BHBJn1W3Z+afd7e5nc5l6z111laVMuc8BqLPMrsRjqiIeBbwJ8DrMvOJpusZNEOiQpn5soXWR8SrgZ8EfqyKZ3k3YbFzHhPTwIae9+uBRxqqRRWKiBV0AuKezPxw0/VUweamhkTEduANwI7M/HrT9WigDgGbI2JTRKyk86z2/Q3XpAGLiADeAzycmb/TdD1VMSSa87+B1cDHI+KhiHh30wVVLSJ+OiKmgWuBv4iI+5quqQrdDgm7gPvo3Mz8YGYeabaqakXEHwEPAC+MiOmIuKnpmmrw/cDPAy/tfoYfioj/0nRRg+aIa0lSIa8kJEmFDAlJUiFDQpJUyJCQJBUyJCRJhQwJSVIhQ0KSVMiQkAasO6jqzRHxzoh4tPuchTuarks6Fw6mkwaoO5fPU3Tmb/rvdEYh/zxwB/DdmfnPzVUnLZ0hIQ1QRLwYeAj46cz8s+6yDcA/Ai/KzC80WZ+0VDY3SYN1JXAS+EjPsrXdv72KUOsYEtJgvRiYzMx/71l2JXA8M/+1oZqkc2ZISIN1JfDgvGVX0WmCklrHkJAG6wr6h8T8ZVIrGBLSgETEemANPVcN3QfTvBhDQi3l40ulAcnMac5+vvVm4FkYEmoprySkal0FPJaZJ5ouRDoXhoRULW9aq9UcTCdJKuSVhCSpkCEhSSpkSEiSChkSkqRChoQkqZAhIUkqZEhIkgoZEpKkQv8fm6FQFXZfS/UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create efficiency/fake rate vs. eta\n",
    "pt, eff, faker, err_eff, err_faker = get_eff_faker_vs_feat('eta',features,X_test,y_test,clf_GBDT)\n",
    "\n",
    "plt.errorbar(pt,eff,yerr=err_eff,linestyle='None',fmt='.')\n",
    "plt.xlabel('$\\eta$',fontsize=14)\n",
    "plt.ylabel('Efficiency',fontsize=14)\n",
    "plt.show()\n",
    "\n",
    "plt.errorbar(pt,faker,yerr=err_faker,linestyle='None',fmt='.')\n",
    "plt.xlabel('$\\eta$',fontsize=14)\n",
    "plt.ylabel('Fake rate',fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}