Final stack.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(models, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(models, handle)\n",
    "        \n",
    "def load_model(filename):\n",
    "    with open(filename, 'rb') as handle:\n",
    "        model = pickle.load(handle)\n",
    "    return model    \n",
    "        \n",
    "def augment(x,y,t=2):\n",
    "    xs,xn = [],[]\n",
    "    for i in range(t):\n",
    "        mask = y>0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xs.append(x1)\n",
    "\n",
    "    for i in range(t//2):\n",
    "        mask = y==0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xn.append(x1)\n",
    "    xs = np.vstack(xs)\n",
    "    xn = np.vstack(xn)\n",
    "    ys = np.ones(xs.shape[0])\n",
    "    yn = np.zeros(xn.shape[0])\n",
    "    x = np.vstack([x,xs,xn])\n",
    "    y = np.concatenate([y,ys,yn])\n",
    "    return x,y\n",
    "\n",
    "def lgb_trainer(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in tqdm(skf.split(X.values, y.values)):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = augment(X_train.values, y_train.values)\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def lgb_trainer_no_aug(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in tqdm(skf.split(X.values, y.values)):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = X_train.values, y_train.values\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def lgb_trainer_for_bayesian_optim(X, y, params):\n",
    "        X_tr, X_val, y_tr, y_val  = train_test_split(X, y, test_size = 0.2, random_state=42)    \n",
    "        trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "        test_data = lgb.Dataset(X_val, label=y_val)\n",
    "        model_lgb     = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "        auc = roc_auc_score(y_val, model_lgb.predict(X_val))\n",
    "        return auc\n",
    "\n",
    "def test_f(X, y, models):\n",
    "    preds = pd.DataFrame({})\n",
    "    for i, model in enumerate(models):\n",
    "        preds[str(i)] = model.predict(X)\n",
    "        print(f\"Fold: {i} \\t Score: {roc_auc_score(y, preds[str(i)].values)}\")\n",
    "    averaged_preds = preds.mean(axis=1)\n",
    "    scorre = roc_auc_score(y, averaged_preds)\n",
    "    print(f\"Score: {scorre}\")\n",
    "    return scorre, averaged_preds, preds\n",
    "\n",
    "def augment_uniques(data):\n",
    "    maps = {}\n",
    "    for feature in tqdm(data.columns):\n",
    "        if feature in ['ID_code', 'target']:\n",
    "            continue\n",
    "        a, b = np.unique(data[feature], return_counts=True)\n",
    "        unique_map = dict(zip(a, b))\n",
    "        maps[feature] = unique_map\n",
    "        data[f'count_{feature}'] = data[feature].map(unique_map)\n",
    "    return data, maps  \n",
    "\n",
    "def answer_agument(testdf, maps):\n",
    "    for feature in tqdm(testdf.columns):\n",
    "        if feature in ['ID_code']:\n",
    "            continue\n",
    "        testdf[f'count_{feature}'] = testdf[feature].map(maps[feature])    \n",
    "    return testdf    \n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import lightgbm as lgb\n",
    "import pickle\n",
    "import gc\n",
    "import seaborn as sns\n",
    "import scikitplot as skplt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "ANSWERX = 'MakeAnswersFromThis.csv'\n",
    "X = 'finalX.csv'\n",
    "y = 'finalY.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stupidity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(TRAIN)\n",
    "test = pd.read_csv(TEST)\n",
    "whole = train.append(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "382c2ecede6e4fe7ba42b33938c35a2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=202), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "whole, unique_map = augment_uniques(whole)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindf, testdf = whole[:200000], whole[200000:]\n",
    "y = traindf.target\n",
    "X = traindf.drop(['target', 'ID_code'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74c43aebdf8d4b43b31725eb39b1ac74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=201), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ATEST = 'data/test.csv'\n",
    "test = pd.read_csv(ATEST)\n",
    "dftest = answer_agument(test, unique_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dftest.to_csv('MakeAnswersFromThis.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.to_csv('finalX.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.to_csv('finalY.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_csv(X)\n",
    "y = pd.read_csv(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_10</th>\n",
       "      <th>var_100</th>\n",
       "      <th>var_101</th>\n",
       "      <th>var_102</th>\n",
       "      <th>var_103</th>\n",
       "      <th>var_104</th>\n",
       "      <th>var_105</th>\n",
       "      <th>var_106</th>\n",
       "      <th>...</th>\n",
       "      <th>count_var_90</th>\n",
       "      <th>count_var_91</th>\n",
       "      <th>count_var_92</th>\n",
       "      <th>count_var_93</th>\n",
       "      <th>count_var_94</th>\n",
       "      <th>count_var_95</th>\n",
       "      <th>count_var_96</th>\n",
       "      <th>count_var_97</th>\n",
       "      <th>count_var_98</th>\n",
       "      <th>count_var_99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.9255</td>\n",
       "      <td>-6.7863</td>\n",
       "      <td>2.9252</td>\n",
       "      <td>9.4763</td>\n",
       "      <td>13.3102</td>\n",
       "      <td>26.5376</td>\n",
       "      <td>1.4403</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>6.0454</td>\n",
       "      <td>9.5426</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.5006</td>\n",
       "      <td>-4.1473</td>\n",
       "      <td>-0.4032</td>\n",
       "      <td>-13.6950</td>\n",
       "      <td>8.4068</td>\n",
       "      <td>35.4734</td>\n",
       "      <td>1.7093</td>\n",
       "      <td>15.1866</td>\n",
       "      <td>2.6227</td>\n",
       "      <td>7.3412</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.6093</td>\n",
       "      <td>-2.7457</td>\n",
       "      <td>-0.3249</td>\n",
       "      <td>-0.3939</td>\n",
       "      <td>12.6317</td>\n",
       "      <td>14.8863</td>\n",
       "      <td>1.3854</td>\n",
       "      <td>15.0284</td>\n",
       "      <td>3.9995</td>\n",
       "      <td>5.3683</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>26</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.0604</td>\n",
       "      <td>-2.1518</td>\n",
       "      <td>2.3061</td>\n",
       "      <td>-19.8592</td>\n",
       "      <td>22.5316</td>\n",
       "      <td>18.6129</td>\n",
       "      <td>1.3512</td>\n",
       "      <td>9.3291</td>\n",
       "      <td>4.2835</td>\n",
       "      <td>10.3907</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>62</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.8369</td>\n",
       "      <td>-1.4834</td>\n",
       "      <td>-9.4458</td>\n",
       "      <td>-22.9264</td>\n",
       "      <td>12.3562</td>\n",
       "      <td>17.3410</td>\n",
       "      <td>1.6940</td>\n",
       "      <td>7.1179</td>\n",
       "      <td>5.1934</td>\n",
       "      <td>8.8230</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>65</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     var_0   var_1  var_10  var_100  var_101  var_102  var_103  var_104  \\\n",
       "0   8.9255 -6.7863  2.9252   9.4763  13.3102  26.5376   1.4403  14.7100   \n",
       "1  11.5006 -4.1473 -0.4032 -13.6950   8.4068  35.4734   1.7093  15.1866   \n",
       "2   8.6093 -2.7457 -0.3249  -0.3939  12.6317  14.8863   1.3854  15.0284   \n",
       "3  11.0604 -2.1518  2.3061 -19.8592  22.5316  18.6129   1.3512   9.3291   \n",
       "4   9.8369 -1.4834 -9.4458 -22.9264  12.3562  17.3410   1.6940   7.1179   \n",
       "\n",
       "   var_105  var_106      ...       count_var_90  count_var_91  count_var_92  \\\n",
       "0   6.0454   9.5426      ...                  1            51             3   \n",
       "1   2.6227   7.3412      ...                  1            77             3   \n",
       "2   3.9995   5.3683      ...                  1            72             6   \n",
       "3   4.2835  10.3907      ...                  1            62             2   \n",
       "4   5.1934   8.8230      ...                  1            65             3   \n",
       "\n",
       "   count_var_93  count_var_94  count_var_95  count_var_96  count_var_97  \\\n",
       "0            18             3            20             2             1   \n",
       "1            24             4             3             1             2   \n",
       "2            26             6            17             2             1   \n",
       "3            24             1             4             3             1   \n",
       "4            16             2            18             1             1   \n",
       "\n",
       "   count_var_98  count_var_99  \n",
       "0            14             4  \n",
       "1            15             5  \n",
       "2            22             8  \n",
       "3             9             3  \n",
       "4            15             3  \n",
       "\n",
       "[5 rows x 400 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_df(models_list, X):\n",
    "    i = 0\n",
    "    df_dict = {}\n",
    "    for models in tqdm(models_list):\n",
    "        models = load_model(models)\n",
    "        for model in tqdm(models):\n",
    "            i += 1\n",
    "            key = f\"col_{i}\"\n",
    "            df_dict[key] = model.predict(X)\n",
    "    return pd.DataFrame(df_dict)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97e30025273a48cbaa9af9ef1920db29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68b6f7c83dd04b599857548bdc61ef53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d219b69108654546a8bfa7ded5a6d38c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "867e4ae903a44675983a00ca8d454900",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e9cf2da73194cc8b7afa0df39c41f7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "models = \"ovvverrr.m ovvverrrx2.m ovvverrrx3.m ovvverrrx4.m\".split()\n",
    "Xstack = get_df(models, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xstack.to_csv('Xstack.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dftest = pd.read_csv('MakeAnswersFromThis.csv')\n",
    "dftest.rename(columns=lambda x: 'z' + str(x) if x[0] == 'c' else x, inplace=True)\n",
    "kek= dftest.reindex(sorted(dftest.columns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83e7740b82a04e9d9f4367818a35aae1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3863d400c79b411ab388baed5b984868",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f4f576fb44b43be89e0f17cd8fb27a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ce942fccb5243ff9736dda29f84cd0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2a92f6bfd70446bbcdf726c3a5885c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "adfstack = get_df(models, kek.drop(['ID_code'], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "adfstack.to_csv('adfstack.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let the stacking begin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_test = pd.read_csv('adfstack.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_csv('Xstack.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 40) | (200000, 40) | (200000, 1)\n"
     ]
    }
   ],
   "source": [
    "print(f\"{X.shape} | {final_test.shape} | {y.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = pd.read_csv('finalY.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.rename(columns=lambda x:'y', inplace=True )\n",
    "y.loc[-1] = 0.0\n",
    "y.index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bayes_opt import BayesianOptimization\n",
    "from skopt import BayesSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | baggin... | featur... | learni... | min_da... | min_su... | num_le... | num_th... | reg_alpha | reg_la... |\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.564487\tvalid_1's auc: 0.500683\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.5007  \u001b[0m | \u001b[0m 0.5378  \u001b[0m | \u001b[0m 0.8561  \u001b[0m | \u001b[0m 0.007588\u001b[0m | \u001b[0m 143.8   \u001b[0m | \u001b[0m 6.56    \u001b[0m | \u001b[0m 10.46   \u001b[0m | \u001b[0m 6.232   \u001b[0m | \u001b[0m 0.8662  \u001b[0m | \u001b[0m 0.6011  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.557723\tvalid_1's auc: 0.502136\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m 0.5021  \u001b[0m | \u001b[95m 0.3043  \u001b[0m | \u001b[95m 0.02832 \u001b[0m | \u001b[95m 0.009729\u001b[0m | \u001b[95m 176.5   \u001b[0m | \u001b[95m 7.123   \u001b[0m | \u001b[95m 11.36   \u001b[0m | \u001b[95m 6.734   \u001b[0m | \u001b[95m 0.3042  \u001b[0m | \u001b[95m 0.5248  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.604854\tvalid_1's auc: 0.498882\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 0.4989  \u001b[0m | \u001b[0m 0.4976  \u001b[0m | \u001b[0m 0.2692  \u001b[0m | \u001b[0m 0.006507\u001b[0m | \u001b[0m 79.53   \u001b[0m | \u001b[0m 7.921   \u001b[0m | \u001b[0m 17.82   \u001b[0m | \u001b[0m 7.824   \u001b[0m | \u001b[0m 0.7852  \u001b[0m | \u001b[0m 0.1997  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.545615\tvalid_1's auc: 0.501045\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 0.501   \u001b[0m | \u001b[0m 0.44    \u001b[0m | \u001b[0m 0.5372  \u001b[0m | \u001b[0m 0.001418\u001b[0m | \u001b[0m 145.1   \u001b[0m | \u001b[0m 6.705   \u001b[0m | \u001b[0m 7.277   \u001b[0m | \u001b[0m 9.796   \u001b[0m | \u001b[0m 0.9656  \u001b[0m | \u001b[0m 0.8084  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.611779\tvalid_1's auc: 0.500011\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.5     \u001b[0m | \u001b[0m 0.5868  \u001b[0m | \u001b[0m 0.09693 \u001b[0m | \u001b[0m 0.007158\u001b[0m | \u001b[0m 121.6   \u001b[0m | \u001b[0m 6.22    \u001b[0m | \u001b[0m 22.33   \u001b[0m | \u001b[0m 6.138   \u001b[0m | \u001b[0m 0.9093  \u001b[0m | \u001b[0m 0.2588  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.632315\tvalid_1's auc: 0.501091\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 0.5011  \u001b[0m | \u001b[0m 0.3362  \u001b[0m | \u001b[0m 0.2874  \u001b[0m | \u001b[0m 0.005681\u001b[0m | \u001b[0m 136.5   \u001b[0m | \u001b[0m 6.849   \u001b[0m | \u001b[0m 38.94   \u001b[0m | \u001b[0m 9.101   \u001b[0m | \u001b[0m 0.9395  \u001b[0m | \u001b[0m 0.8948  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.57323\tvalid_1's auc: 0.504519\n",
      "| \u001b[95m 7       \u001b[0m | \u001b[95m 0.5045  \u001b[0m | \u001b[95m 0.3815  \u001b[0m | \u001b[95m 0.8305  \u001b[0m | \u001b[95m 0.001796\u001b[0m | \u001b[95m 87.44   \u001b[0m | \u001b[95m 5.452   \u001b[0m | \u001b[95m 16.39   \u001b[0m | \u001b[95m 7.555   \u001b[0m | \u001b[95m 0.2713  \u001b[0m | \u001b[95m 0.8287  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.556684\tvalid_1's auc: 0.498525\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.4985  \u001b[0m | \u001b[0m 0.5503  \u001b[0m | \u001b[0m 0.26    \u001b[0m | \u001b[0m 0.005884\u001b[0m | \u001b[0m 79.73   \u001b[0m | \u001b[0m 13.02   \u001b[0m | \u001b[0m 7.609   \u001b[0m | \u001b[0m 9.948   \u001b[0m | \u001b[0m 0.7722  \u001b[0m | \u001b[0m 0.1987  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.558624\tvalid_1's auc: 0.496772\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.4968  \u001b[0m | \u001b[0m 0.7961  \u001b[0m | \u001b[0m 0.7358  \u001b[0m | \u001b[0m 0.007362\u001b[0m | \u001b[0m 162.1   \u001b[0m | \u001b[0m 12.71   \u001b[0m | \u001b[0m 7.592   \u001b[0m | \u001b[0m 7.434   \u001b[0m | \u001b[0m 0.1159  \u001b[0m | \u001b[0m 0.8631  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.616915\tvalid_1's auc: 0.50352\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.5035  \u001b[0m | \u001b[0m 0.3637  \u001b[0m | \u001b[0m 0.3045  \u001b[0m | \u001b[0m 0.001572\u001b[0m | \u001b[0m 103.5   \u001b[0m | \u001b[0m 8.252   \u001b[0m | \u001b[0m 30.54   \u001b[0m | \u001b[0m 8.55    \u001b[0m | \u001b[0m 0.8872  \u001b[0m | \u001b[0m 0.4722  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.634973\tvalid_1's auc: 0.499773\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.4998  \u001b[0m | \u001b[0m 0.7163  \u001b[0m | \u001b[0m 0.6448  \u001b[0m | \u001b[0m 0.007847\u001b[0m | \u001b[0m 138.6   \u001b[0m | \u001b[0m 12.71   \u001b[0m | \u001b[0m 22.28   \u001b[0m | \u001b[0m 8.091   \u001b[0m | \u001b[0m 0.4275  \u001b[0m | \u001b[0m 0.02542 \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.611698\tvalid_1's auc: 0.503329\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.5033  \u001b[0m | \u001b[0m 0.7245  \u001b[0m | \u001b[0m 0.03797 \u001b[0m | \u001b[0m 0.006728\u001b[0m | \u001b[0m 104.0   \u001b[0m | \u001b[0m 10.09   \u001b[0m | \u001b[0m 36.76   \u001b[0m | \u001b[0m 6.997   \u001b[0m | \u001b[0m 0.4104  \u001b[0m | \u001b[0m 0.7556  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.634529\tvalid_1's auc: 0.504991\n",
      "| \u001b[95m 13      \u001b[0m | \u001b[95m 0.505   \u001b[0m | \u001b[95m 0.6398  \u001b[0m | \u001b[95m 0.07851 \u001b[0m | \u001b[95m 0.003608\u001b[0m | \u001b[95m 82.57   \u001b[0m | \u001b[95m 14.3    \u001b[0m | \u001b[95m 33.28   \u001b[0m | \u001b[95m 8.534   \u001b[0m | \u001b[95m 0.8715  \u001b[0m | \u001b[95m 0.8037  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.598634\tvalid_1's auc: 0.501885\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 0.5019  \u001b[0m | \u001b[0m 0.6694  \u001b[0m | \u001b[0m 0.8044  \u001b[0m | \u001b[0m 0.005854\u001b[0m | \u001b[0m 173.0   \u001b[0m | \u001b[0m 13.96   \u001b[0m | \u001b[0m 16.13   \u001b[0m | \u001b[0m 6.44    \u001b[0m | \u001b[0m 0.2279  \u001b[0m | \u001b[0m 0.4271  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.552345\tvalid_1's auc: 0.495322\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 0.4953  \u001b[0m | \u001b[0m 0.2274  \u001b[0m | \u001b[0m 0.7761  \u001b[0m | \u001b[0m 0.001063\u001b[0m | \u001b[0m 131.5   \u001b[0m | \u001b[0m 9.174   \u001b[0m | \u001b[0m 12.77   \u001b[0m | \u001b[0m 6.479   \u001b[0m | \u001b[0m 0.3376  \u001b[0m | \u001b[0m 0.9429  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.653007\tvalid_1's auc: 0.504234\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 0.5042  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.5034  \u001b[0m | \u001b[0m 0.003547\u001b[0m | \u001b[0m 199.9   \u001b[0m | \u001b[0m 5.383   \u001b[0m | \u001b[0m 38.16   \u001b[0m | \u001b[0m 9.597   \u001b[0m | \u001b[0m 0.4778  \u001b[0m | \u001b[0m 0.572   \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.671431\tvalid_1's auc: 0.503337\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 0.5033  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.7767  \u001b[0m | \u001b[0m 0.008392\u001b[0m | \u001b[0m 60.88   \u001b[0m | \u001b[0m 5.271   \u001b[0m | \u001b[0m 37.94   \u001b[0m | \u001b[0m 9.625   \u001b[0m | \u001b[0m 0.5532  \u001b[0m | \u001b[0m 0.3889  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[95]\ttraining's auc: 0.537152\tvalid_1's auc: 0.50114\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 0.5011  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.851   \u001b[0m | \u001b[0m 0.008189\u001b[0m | \u001b[0m 199.1   \u001b[0m | \u001b[0m 7.667   \u001b[0m | \u001b[0m 5.131   \u001b[0m | \u001b[0m 9.873   \u001b[0m | \u001b[0m 0.249   \u001b[0m | \u001b[0m 0.3354  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.64568\tvalid_1's auc: 0.503343\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 0.5033  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.1239  \u001b[0m | \u001b[0m 0.00132 \u001b[0m | \u001b[0m 199.5   \u001b[0m | \u001b[0m 5.442   \u001b[0m | \u001b[0m 39.81   \u001b[0m | \u001b[0m 9.465   \u001b[0m | \u001b[0m 0.4562  \u001b[0m | \u001b[0m 0.1286  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.661667\tvalid_1's auc: 0.500398\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 0.5004  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.861   \u001b[0m | \u001b[0m 0.007125\u001b[0m | \u001b[0m 199.3   \u001b[0m | \u001b[0m 5.128   \u001b[0m | \u001b[0m 39.99   \u001b[0m | \u001b[0m 7.134   \u001b[0m | \u001b[0m 0.4101  \u001b[0m | \u001b[0m 0.3608  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.626784\tvalid_1's auc: 0.497265\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 0.4973  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.8345  \u001b[0m | \u001b[0m 0.009456\u001b[0m | \u001b[0m 170.0   \u001b[0m | \u001b[0m 5.239   \u001b[0m | \u001b[0m 26.32   \u001b[0m | \u001b[0m 9.758   \u001b[0m | \u001b[0m 0.5331  \u001b[0m | \u001b[0m 0.1845  \u001b[0m |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[97]\ttraining's auc: 0.650741\tvalid_1's auc: 0.501416\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 0.5014  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.1508  \u001b[0m | \u001b[0m 0.004867\u001b[0m | \u001b[0m 198.5   \u001b[0m | \u001b[0m 14.63   \u001b[0m | \u001b[0m 36.97   \u001b[0m | \u001b[0m 7.29    \u001b[0m | \u001b[0m 0.9188  \u001b[0m | \u001b[0m 0.6807  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\ttraining's auc: 0.676347\tvalid_1's auc: 0.50331\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m 0.5033  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.5493  \u001b[0m | \u001b[0m 0.004028\u001b[0m | \u001b[0m 60.88   \u001b[0m | \u001b[0m 5.183   \u001b[0m | \u001b[0m 39.81   \u001b[0m | \u001b[0m 8.606   \u001b[0m | \u001b[0m 0.6437  \u001b[0m | \u001b[0m 0.01046 \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.66808\tvalid_1's auc: 0.501472\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m 0.5015  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.1168  \u001b[0m | \u001b[0m 0.006785\u001b[0m | \u001b[0m 60.34   \u001b[0m | \u001b[0m 5.236   \u001b[0m | \u001b[0m 39.71   \u001b[0m | \u001b[0m 9.911   \u001b[0m | \u001b[0m 0.7693  \u001b[0m | \u001b[0m 0.2853  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.655668\tvalid_1's auc: 0.500385\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m 0.5004  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.441   \u001b[0m | \u001b[0m 0.002685\u001b[0m | \u001b[0m 63.41   \u001b[0m | \u001b[0m 14.91   \u001b[0m | \u001b[0m 39.84   \u001b[0m | \u001b[0m 9.476   \u001b[0m | \u001b[0m 0.01056 \u001b[0m | \u001b[0m 0.3424  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.677144\tvalid_1's auc: 0.50237\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m 0.5024  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.5411  \u001b[0m | \u001b[0m 0.005579\u001b[0m | \u001b[0m 64.25   \u001b[0m | \u001b[0m 5.861   \u001b[0m | \u001b[0m 39.9    \u001b[0m | \u001b[0m 6.021   \u001b[0m | \u001b[0m 0.1498  \u001b[0m | \u001b[0m 0.4206  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.658273\tvalid_1's auc: 0.500012\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m 0.5     \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.2604  \u001b[0m | \u001b[0m 0.003212\u001b[0m | \u001b[0m 62.15   \u001b[0m | \u001b[0m 5.603   \u001b[0m | \u001b[0m 39.93   \u001b[0m | \u001b[0m 9.478   \u001b[0m | \u001b[0m 0.6766  \u001b[0m | \u001b[0m 0.6751  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\ttraining's auc: 0.66897\tvalid_1's auc: 0.502095\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m 0.5021  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.6988  \u001b[0m | \u001b[0m 0.006264\u001b[0m | \u001b[0m 199.9   \u001b[0m | \u001b[0m 5.305   \u001b[0m | \u001b[0m 40.0    \u001b[0m | \u001b[0m 6.257   \u001b[0m | \u001b[0m 0.06556 \u001b[0m | \u001b[0m 0.3264  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.637788\tvalid_1's auc: 0.504633\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m 0.5046  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.0639  \u001b[0m | \u001b[0m 0.002958\u001b[0m | \u001b[0m 199.1   \u001b[0m | \u001b[0m 5.115   \u001b[0m | \u001b[0m 39.9    \u001b[0m | \u001b[0m 8.984   \u001b[0m | \u001b[0m 0.0591  \u001b[0m | \u001b[0m 0.5729  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.661836\tvalid_1's auc: 0.503018\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m 0.503   \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.5046  \u001b[0m | \u001b[0m 0.004365\u001b[0m | \u001b[0m 199.5   \u001b[0m | \u001b[0m 5.294   \u001b[0m | \u001b[0m 39.9    \u001b[0m | \u001b[0m 9.993   \u001b[0m | \u001b[0m 0.4617  \u001b[0m | \u001b[0m 0.4803  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.658815\tvalid_1's auc: 0.499299\n",
      "| \u001b[0m 31      \u001b[0m | \u001b[0m 0.4993  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.4662  \u001b[0m | \u001b[0m 0.003986\u001b[0m | \u001b[0m 197.8   \u001b[0m | \u001b[0m 5.753   \u001b[0m | \u001b[0m 39.88   \u001b[0m | \u001b[0m 9.282   \u001b[0m | \u001b[0m 0.3502  \u001b[0m | \u001b[0m 0.3289  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.688466\tvalid_1's auc: 0.504898\n",
      "| \u001b[0m 32      \u001b[0m | \u001b[0m 0.5049  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.2061  \u001b[0m | \u001b[0m 0.009843\u001b[0m | \u001b[0m 60.39   \u001b[0m | \u001b[0m 5.36    \u001b[0m | \u001b[0m 39.64   \u001b[0m | \u001b[0m 6.727   \u001b[0m | \u001b[0m 0.3092  \u001b[0m | \u001b[0m 0.4968  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\ttraining's auc: 0.670773\tvalid_1's auc: 0.501574\n",
      "| \u001b[0m 33      \u001b[0m | \u001b[0m 0.5016  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.3466  \u001b[0m | \u001b[0m 0.003555\u001b[0m | \u001b[0m 63.25   \u001b[0m | \u001b[0m 5.326   \u001b[0m | \u001b[0m 39.51   \u001b[0m | \u001b[0m 6.906   \u001b[0m | \u001b[0m 0.7413  \u001b[0m | \u001b[0m 0.3715  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.675294\tvalid_1's auc: 0.504771\n",
      "| \u001b[0m 34      \u001b[0m | \u001b[0m 0.5048  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.4568  \u001b[0m | \u001b[0m 0.004256\u001b[0m | \u001b[0m 62.46   \u001b[0m | \u001b[0m 5.049   \u001b[0m | \u001b[0m 39.85   \u001b[0m | \u001b[0m 6.643   \u001b[0m | \u001b[0m 0.3906  \u001b[0m | \u001b[0m 0.8568  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.677559\tvalid_1's auc: 0.503726\n",
      "| \u001b[0m 35      \u001b[0m | \u001b[0m 0.5037  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.3224  \u001b[0m | \u001b[0m 0.006015\u001b[0m | \u001b[0m 66.88   \u001b[0m | \u001b[0m 5.029   \u001b[0m | \u001b[0m 39.83   \u001b[0m | \u001b[0m 6.555   \u001b[0m | \u001b[0m 0.5979  \u001b[0m | \u001b[0m 0.2049  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.677648\tvalid_1's auc: 0.506999\n",
      "| \u001b[95m 36      \u001b[0m | \u001b[95m 0.507   \u001b[0m | \u001b[95m 0.8     \u001b[0m | \u001b[95m 0.6409  \u001b[0m | \u001b[95m 0.004444\u001b[0m | \u001b[95m 61.13   \u001b[0m | \u001b[95m 5.477   \u001b[0m | \u001b[95m 39.8    \u001b[0m | \u001b[95m 6.977   \u001b[0m | \u001b[95m 0.5714  \u001b[0m | \u001b[95m 0.766   \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.549205\tvalid_1's auc: 0.499602\n",
      "| \u001b[0m 37      \u001b[0m | \u001b[0m 0.4996  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.7717  \u001b[0m | \u001b[0m 0.009138\u001b[0m | \u001b[0m 60.38   \u001b[0m | \u001b[0m 5.133   \u001b[0m | \u001b[0m 5.973   \u001b[0m | \u001b[0m 6.201   \u001b[0m | \u001b[0m 0.9791  \u001b[0m | \u001b[0m 0.6875  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.675045\tvalid_1's auc: 0.502578\n",
      "| \u001b[0m 38      \u001b[0m | \u001b[0m 0.5026  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.4471  \u001b[0m | \u001b[0m 0.004979\u001b[0m | \u001b[0m 61.37   \u001b[0m | \u001b[0m 5.758   \u001b[0m | \u001b[0m 39.99   \u001b[0m | \u001b[0m 6.417   \u001b[0m | \u001b[0m 0.1202  \u001b[0m | \u001b[0m 0.3004  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.681648\tvalid_1's auc: 0.503585\n",
      "| \u001b[0m 39      \u001b[0m | \u001b[0m 0.5036  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.3006  \u001b[0m | \u001b[0m 0.0066  \u001b[0m | \u001b[0m 66.37   \u001b[0m | \u001b[0m 5.32    \u001b[0m | \u001b[0m 39.78   \u001b[0m | \u001b[0m 6.257   \u001b[0m | \u001b[0m 0.3018  \u001b[0m | \u001b[0m 0.8878  \u001b[0m |\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.68226\tvalid_1's auc: 0.504263\n",
      "| \u001b[0m 40      \u001b[0m | \u001b[0m 0.5043  \u001b[0m | \u001b[0m 0.8     \u001b[0m | \u001b[0m 0.6631  \u001b[0m | \u001b[0m 0.00549 \u001b[0m | \u001b[0m 63.86   \u001b[0m | \u001b[0m 5.165   \u001b[0m | \u001b[0m 39.73   \u001b[0m | \u001b[0m 6.304   \u001b[0m | \u001b[0m 0.8974  \u001b[0m | \u001b[0m 0.2275  \u001b[0m |\n",
      "=====================================================================================================================================\n"
     ]
    }
   ],
   "source": [
    "def bayes_parameter_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=6, n_estimators=10000, output_process=False):\n",
    "    # prepare data\n",
    "#     Xtr, Xval, ytr, yval  = train_test_split(X, y, test_size = 0.2, random_state=42)\n",
    "#     train_data = lgb.Dataset(data=Xtr, label=ytr, free_raw_data=False)\n",
    "#     val_data = lgb.Dataset(data=Xval, label=yval, free_raw_data=False)\n",
    "\n",
    "    # parameters\n",
    "    def lgb_eval(num_leaves, feature_fraction, bagging_fraction, learning_rate, num_threads, min_data_in_leaf, min_sum_hessian_in_leaf, reg_alpha, reg_lambda):\n",
    "        # fixed parameters\n",
    "        params = {'application':'binary',\n",
    "                  'num_iterations': n_estimators,\n",
    "                  'learning_rate':learning_rate,\n",
    "                  'early_stopping_round':100,\n",
    "                  'metric':'auc',\n",
    "                  'max_depth':-1,\n",
    "                  'bagging_freq':7,\n",
    "                  'verbosity':-1}\n",
    "        # variables\n",
    "        params[\"num_leaves\"] = int(round(num_leaves))\n",
    "        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n",
    "        params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n",
    "        params['learning_rate'] = learning_rate\n",
    "        params['num_threads'] = int(num_threads)\n",
    "        params['min_data_in_leaf'] = int(min_data_in_leaf)\n",
    "        params['min_sum_hessian_in_leaf'] = min_sum_hessian_in_leaf\n",
    "        params['reg_alpha'] = max(min(reg_alpha, 1), 0)\n",
    "        params['reg_lambda'] = max(min(reg_lambda, 1), 0)\n",
    "        \n",
    "        score = lgb_trainer_for_bayesian_optim(X, y, params)\n",
    "#         cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, stratified=True, verbose_eval =200, metrics=['auc'])\n",
    "        \n",
    "        return score\n",
    "    # range of variables\n",
    "    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (5, 40),\n",
    "                                            'feature_fraction': (0.01, 0.9),\n",
    "                                            'bagging_fraction': (0.8, 0.1),\n",
    "                                            'learning_rate': (0.001, 0.01),\n",
    "                                            'num_threads': (6, 10),\n",
    "                                            'min_data_in_leaf': (60, 200),\n",
    "                                            'min_sum_hessian_in_leaf': (5.0 , 15.0),\n",
    "                                            'reg_alpha': (0.0 , 1.0),\n",
    "                                            'reg_lambda': (0.0 , 1.0),\n",
    "                                           },\n",
    "                                             random_state=42)\n",
    "    # optimize!\n",
    "    lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n",
    "    \n",
    "    # output optimization process\n",
    "    if output_process==True: lgbBO.points_to_csv(\"bayes_opt_result.csv\")\n",
    "    \n",
    "    # return best parameters\n",
    "    return lgbBO.res\n",
    "\n",
    "opt_params = bayes_parameter_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=42, n_estimators=100) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'target': 0.5069993743091272,\n",
       " 'params': {'bagging_fraction': 0.8,\n",
       "  'feature_fraction': 0.6409129751307899,\n",
       "  'learning_rate': 0.004444183166341189,\n",
       "  'min_data_in_leaf': 61.12700472473412,\n",
       "  'min_sum_hessian_in_leaf': 5.477195782834263,\n",
       "  'num_leaves': 39.80431753787029,\n",
       "  'num_threads': 6.977054472760379,\n",
       "  'reg_alpha': 0.5713746629098466,\n",
       "  'reg_lambda': 0.7659697399878922}}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_params[35]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xval, ytr, yval  = train_test_split(X, y, test_size = 0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "   'bagging_fraction': 0.8,\n",
    "  'feature_fraction': 0.6409129751307899,\n",
    "  'learning_rate': 0.004444183166341189,\n",
    "  'min_data_in_leaf': 61,\n",
    "  'min_sum_hessian_in_leaf': 5.477195782834263,\n",
    "  'num_leaves': 39,\n",
    "  'num_threads': 7,\n",
    "  'reg_alpha': 0.5713746629098466,\n",
    "  'reg_lambda': 0.7659697399878922,\n",
    "    'boost': 'gbdt',\n",
    "    'metric':'auc',\n",
    "    'tree_learner': 'serial',\n",
    "    'objective': 'binary', \n",
    "    'verbosity': -1,\n",
    "    'seed':42,\n",
    "}\n",
    "\n",
    "\n",
    "#^best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8881b5a93bcb4333b8a643cb305bc058",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[2]\ttraining's auc: 0.537037\tvalid_1's auc: 0.503822\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[28]\ttraining's auc: 0.58447\tvalid_1's auc: 0.501803\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[44]\ttraining's auc: 0.594663\tvalid_1's auc: 0.505405\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[7]\ttraining's auc: 0.55323\tvalid_1's auc: 0.509027\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[13]\ttraining's auc: 0.593732\tvalid_1's auc: 0.502715\n"
     ]
    }
   ],
   "source": [
    "rank_models = lgb_trainer_no_aug(Xtr, pd.Series(ytr.values[:,0]), param, n_folds = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.5017701249486\n",
      "Fold: 1 \t Score: 0.4995392315914421\n",
      "Fold: 2 \t Score: 0.5055092692410392\n",
      "Fold: 3 \t Score: 0.5023224748657927\n",
      "Fold: 4 \t Score: 0.5004523109329986\n",
      "Score: 0.5050976123548913\n"
     ]
    }
   ],
   "source": [
    "scorre, q, w = test_f(Xval, yval, rank_models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# asdasd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_test = pd.read_csv('adfstack.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.103566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.207013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.157158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.188586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.045161</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    target\n",
       "0  test_0  0.103566\n",
       "1  test_1  0.207013\n",
       "2  test_2  0.157158\n",
       "3  test_3  0.188586\n",
       "4  test_4  0.045161"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.read_csv('answer_1_simple_gbm.csv')\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.target = final_test.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.044425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.129835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.151489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.174534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.053164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    target\n",
       "0  test_0  0.044425\n",
       "1  test_1  0.129835\n",
       "2  test_2  0.151489\n",
       "3  test_3  0.174534\n",
       "4  test_4  0.053164"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.to_csv('#a1.csv', index=False)\n",
    "submission.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cOrr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f63bb337be0>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAESCAYAAADzBx6nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnXmcXFWZv5+3O+nsCySBhASSgGwBI4SI4IALiiIzw6KjA66II/BTwEFQQRYji+w4MoAaMRAZN8QRUVFABnFXwr5vJiQN2UMCSWfp7np/f9zboeic9/St6tvVtyrvk8/9pPqcOve8devWqVPnfc/3FVXFcRzHaVya+tsAx3Ecp2/xgd5xHKfB8YHecRynwfGB3nEcp8Hxgd5xHKfB8YHecRynwfGB3nEcp8aIyBwRWSYijxn1IiJXi8hzIvKIiMwoq/uEiDybHp/I0p8P9I7jOLXnRuCwSP37gF3T4wTgmwAisi3wFeAtwP7AV0Rkm54684HecRynxqjq74FVkaccCXxPE/4KjBaRCcB7gbtUdZWqvgzcRfwLA4ABeRjdH7Sv+Ie5pXf+wZ+p+HybNtqXoknCXQ1o6ay4n9j5Sipmm84O+zv5yNXLguXDmgeZbZZtXG3WbSp1mHV7Dp8ULH94zQKzzeTh25l1i9eH7/XtB/c4SQnyakdbsHzUwGFmm2Ub7GuxesM6s27fMbsEyx95eb7ZZsfh48y6jZ2bguUDmux7s1ns+2LMwBHB8rbOjWabGGs7NwTLx7aMNNssjdxngn2/N0m4bkhzi9lmTbv9Xr2w8hG7s4zExpzutIzb5USSmXgXs1V1dgXdTQQWlf3dmpZZ5VFqMqMXkVkickak/oMi8riIlERkZi1schzH6StUdbaqziw7Khnkc6coSzePAe8Hft/fhjiO4wTpbM9+9J4XgR3L/p6UllnlUXo10IvIx1OP8MMicpOITBGR/0vL7haRnbKcR1WfVNWnM/R3gojME5F513/vh70x3XEcpzJKpexH77kN+HgafXMAsEZVFwN3AO8RkW1SJ+x70rIoVa/Ri8hewDnAW1V1ReoNngvMVdW5InI8cDVwVLV9dCf9+TMbKlsvcxzH6S2quQzgAIjID4F3AGNFpJUkkmZg0o9+C7gdOBx4DmgDPpnWrRKRC4D70lOdr6oxpy7QO2fsIcBPVHVFmQEHkizBANwEXNaL80eJOVyn/uG6YPmid55ktunYZH9vNA8Mv8HNzdV914jhjG2iuvMtaQu/zweN2cNss3zTmqr6sojJXUedbpG6ali6Luz822Oc7a9avqG6a1GN7bFrIRHHajW8tGFlsHyPoTuYbZZ2vFpxPys2vcJ2LaMqbld35DNTB0BVj+2hXoHPGnVzgDmV9Fe3UTeO4xSDrWKQB8hxRl9rejOF+D/ggyIyBjYH8v8ZOCat/wjwh96Z5ziOUxBKndmPglH1jF5VHxeRi4B7RaQTeBA4BbhBRL4ALCddV+oJETka+G9gHPArEXlIVd9brW2O4zi502nvLyk6vVq6UdW5JA7Ycg4JPG9WD+f5GfCz3tjiOI7Tl+TpjK01vkbvOI6ThRydsbWmpgO9iFwL/FO34m+o6g159mNF1+x4z7fMNk/vf6pZN2hA+A1uaq7ujddSOPJCmuzIFUs2AaBkRLwsaX/FbDN+kC0xsGDdUrMutuW+GkpVRhpZqHG+ZZFrsd3g0Wbdqg12FEreEUMW1vsL0FyFCbHImvEDbTmDVzvWh22o8jpYMgfVEov8yqcDH+ijiMgsYK2qBsOF0rjQI4ESsAw4TlVfqoVtjuM4mSigkzUrRZFAuFxVp6vqPsAvgfP62yDHcZzX0dmR/SgYRZFAKP9dPQzCv7/LJRB+vHpR6CmO4zh9g5ayHwWjMBIIaajmx4E1wDtDzymXQHhmz8NcAsFxnNqxlTpjc5VAUNWzgbNF5CzgZBLtB5OYfrwlZxBzuO7+96vNugemhxWWBw20f6I1RxyrlgRCR6f9AytW19wUrutQe01x5UbbOTlx6Fizbn0prMwnEcdazA7LodkZmRVZDlewJQY2GHYDrIpdi+GxaxHWj48RuxadRl3M6dtexdgTyzfQuulls25CS9hp3Ra5DrGQxFIVTtz2yDp57B7MA428d0WnKGv05Xwf+EB/G+E4jvM66njpphASCCKya9mfRwJP9cIux3Gc/KmtTHGuFEICAbhERHYnCa98AbBlJh3HcfqDfBKK9AtFkUDwpRrHcYpNAZdkslK3EgixnaKWfry1wxVshyvAjEeuMOtefPeJwfLYLteN68OXvQUYMSacuDl2vg1/DzvDnl7dyvRtpwbrYs7OBWvtnbFHjd0nWP5gx/Nmm0FNA826TUbM8ZK2VdFE2hadhrPuudUvss/YcDLvmDN24SvhxOsAB++wc7D80dICs00suXXsPRkgzcHylkjicCtR+qsdbWw3KOxYje0ubd1o57d4x/Dwtf1tJDl4k/GawHa4x17vK+197Cwt4JJMVhpSAqFWWIN8tViDfLVYg3w9UM0gH8Ma5OsBa5CvFmuQrxZrkG84fEYfJ4MEwizg0yTr+gBfVtXba2Gb4zhOJnxGnwtfV1V7jcRxHKc/qeOBvhASCBX0t1kC4eZXFuZ5asdxnCja2Z75KBpVD/RlEgiHqOqbgM+RZImaq6rTSTY+2dtNt+Tk9AtijogENXRVdbaqzlTVmR8amet3iOM4Tpw63jBVFAmEbwIXkIiZXQBcCRwfazCgxfawNzeHPfYx/fiYnIHldJ3422+bbV56zwlm3QAj+qd9Q3XfuwOaws66WBRHTFc+tuW+1dAyj0kFxCI5LPmGmAZ7TMfc2gbfUeXW+dim+hc61gTLxw+ztf5j70k1bWJyEHnaALa8xHzjOgDsMHiMWbd8kx3tZPUVe71550rYgjpeuinEGr2qbo7nE5HvkEgVO47jFIcCztSzUhQJhAllfx4NPNYLuxzHcfLHJRB6LYFwmYjsQ7J0swDIN0DdcRyntxQwoUhWiiKB8LHe2OE4jtPnFHCmnpVCrNEXgah+vFEXc7jucOdss85KXh6T0y51Vq61/fSaVnYfNSl8voizsxpd72Vt9lb33ceMN+sWbd4jl528bY9RzflWrLedjDtvu71Z90LnimB5zPkcdVrXKHn5wg0r2clwui7bZDtqJ0QS1C+JSCf0G3W8Rl/TnbHAVAISCMBhwO7p36OB1Wn+WKcXWIO84+SJNcg3HD6jz4YlgQBs1roRkStJ0gk6juMUhzqe0RdqZ6wkv5M/BPywN3Y5juPkTh1H3RRpZyzAwcBSVX3W6HOzBMKPVy+q1nTHcZzK6ezIfhSM3szot9gZCxwI/CCtvwk4qMJzHktkNl8ugfDvo3eswmTHcZwqqeMZfWGibkRkAIl8wn5Znh9LPCJGnZZiW+crTxRiSRmAHVkDsOM93wqWt77L3j5Q6rS/k6vZ+m1JDwA0Ra5TJ+HXHIsMWdJuR6GMH7KtWWcRlUCoItIk1qYpcm07IxEvFssi12K7llHB8pXtYdmJnpCcJQGs616tpEIsImcHIyJnTWc4mQrkH3G1BVW830WhEDtjU94NPKWqrb2wyXEcp2/IcUYvIoeJyNMi8pyInBmon5z6OR8Rkd+JyKSyup1E5E4ReVJEnhCRKT31V5SdsZB8QbgT1nGcYpLTkoyINAPXAocCrcB9InKbqj5R9rQrgO+p6lwROQS4GOjaWPo94CJVvUtEhoPxM7uMQuyMTZ9zXG9scRzH6VPyC6/cH3hOVf8BICI/Ao4Eygf6acDn08f3ALemz50GDFDVuwBUdW2WDvtY19NxHKdB6OzMfJRHCKZH+Tb6iUB52GBrWlbOw7wm+X40MCJdJt8NWC0i/ysiD4rI5ekvhCh1mxy8pBEHmqFZbUkZAHREnJ1jjKTdMf34mF/IcrpOutvWt19+xKfMOssZFnPSxjTiY9vqB8nAimyAuJNsSduqYPk+o+zE5ssjzklLr9zS7I+1AShFXteQpvC1iF2/2Hti6bOPjyTzXtNhOye1ivuiGhZvWs3Og7eryAYAIna8tPHlYPkuQ8L9AKzelGlyWz0VLN2o6mzA1kHpmTOAa0TkOOD3wItAJ8mYfTCwL7AQ+DFwHPDd2MmKkhz8TcC3gOEk6pUfUVU7PMFxnMJgDfINR35hky8C5fHhk9KyzajqS6Qz+nQd/gOqulpEWoGHypZ9bgUOoIeBvihLN9cDZ6rqG4GfAV/oZ3scx3FeT36pBO8DdhWRqSLSQhKIclv5E0RkrLwWH3sWMKes7WgRGZf+fQivX9sPUhQJhN1Ifp4A3AV8wOjPd8Y6jtMvaEkzH9HzqHYAJwN3AE8CN6dRjOeLyBHp094BPC0izwDbAxelbTtJlnXuFpFHSbJdfqcn26teuimTQHirqq5I4+jnkkggzBWR40kkEI7KcLrHSbzOtwIf5PU/azZTvu71zJ6H1e/uBcdx6o8cpQ1U9Xbg9m5l55U9vgW4xWh7FzC9kv6KIoFwPPAZEbkfGAFs6oVdjuM4+VPS7EfBKIQEgqo+BbwHQER2A/65pzadHZV/R8VkE2JRN7FoHYtYohBLziAWWTPuNtvX0jHp7cHyWGRNLEqmmi3tnRFHVTURPksi2+PHG1IBAE9U4TCrNgIp7/NZ0T8xqYCdBtla8M+2rwv3E7MvEoFk3RexT2I154uxNCIhsU3LiIrPVxEF1LDJSiEkEERku/T/JpLloLAYjOM4Tn+xNYqa5SyBcKyIdIVe/i9liUgcx3EKQR2LmhVCAkFVv0GSUtBxHKeYFHCmnpVCrNE7juMUns7O/ragaiTmmMm9sxwlEHYbN9M03NpWH92aHtFn39ARDgKKbauPYW1BjzmnOkr2Tba29d5g+Rkzv2y2iTlnmiP67KtpD5YPi8wZYhIDA4y+YndlZ6T2gneuDJaffo8tI9BSpatqpYbvi22lxWwTc07GrrtFe+R8X9t1WbD8q8+ON9vE7BtK+H5/tmRLUuzSNNysi9FkXIsNEaHGwZH38YoFP+y1WH3bpZ/MPFgO/dINfSyOXxm57owVkVkickbkKb8DBpLEgP6Hqu6jqjeIyBgRuUdE1orINXna5DiOkwdaKmU+ikatl24eI9Fv6K7etQE4F9g7PRzHcYpFAePjs5JpRp+X1IGqPqmqTwfK16nqH0kG/JgdmyUQ1mxYnqVLx3GcfMhP66bm9Dijz1nqoFeUSyDE1ugdx3Fyp6N+nbFZZvR5Sh04juPUJy6BUHuGNQ8y6w4as0ewfElk+3SH2t/Wg4wEE9Vs4Y5RbaIQK7rminlfM9t8deY5Zt2GSOTFzM7BwfLHmsPROAC7lMLXD2BJU/i671CqLqLp3HvCkgBfP8yWEbj0N9uadUuNKCOAt2o4ouR5sdtM0lhETuUMjIwp1z4zKVh+wf4vBssBbvxbuA3Aq4aEyMHYEU2rI69qQCR5kPXuT1D7vlhThVRJRRRwSSYrWWb0uUkdOI7j1C2NPKPPU+pARI4G/hsYB/xKRB5S1femdQuAkUCLiBwFvKdbVnTHcZx+o4hhk1nJtHSTo9TBz0gySIXqpmSxxXEcp18o4Ew9K3W7Ru84jlNTXAKh20lzlDqwmLjNXqbhImEnz/hB25jnW7whLJsAttM15jytRm6hWv34Y0e/KVg+LOKC+cq8C826s2aebdZZrrBNEQduzA6rXWw7e0wCYQ3hD2PsfOe9a4VZ96W77XvGciVasg492VENscUE672y5AUATp6wxKy7ZnFYOmF5JE/QOGzncwzrKnVExTFsLs1BAmHt54/I3Pnwq24rlARCrjN6EZkFrFXVzxr1HwRmAXsC+6vqvLR8f9L4eJLPz6x0mcdxHKcQ9JQLtsgURQLhMWCmqnaIyATgYRH5RZpE13Ecp/+p44G+KBIIbWWD+mAM8cJyCYR1G1/O0qXjOE4+1HGGqR4H+jIJhENU9U3A50hCJOeq6nTg+yQSCL1CRN4iIo8DjwInhWbzqjpbVWeq6sxhkfV2x3Gc3GnkOHoCEggiciDJEgwkEgiX9dYQVf0bsJeI7AnMFZFfq6opcrapVPmqzoJ1S826iUPH2u3WhtvFnFqWQxigqRSuizlwY85YS8c8tsM15nC9eN5FZt1pM88Klg+MXIu22O5Io93GSJuYBrs1c1lnOGkBzrnb3hl76aH2L8dT7gonox4R+VjFroV1DWPzw/ZI7QjDHRuz4YrF25l1Z+9u76j97FPhide2Yu+KjhO+FrH3PuYEzwPtLN5MPSv5hgDkgKo+CazF5Yodpy6wBvmGo45n9IWQQBCRqSIyIH08GdgDWNCbczqO4+RKHQ/0RZFAOAg4U0TaSX6pfqZrqchxHKcINHx4ZV9LIKjqTSRr/Y7jOMWk0Qd6x3GcrR3t8IH+ddRCAmHP4bZutkVMsmB9ydYQP2rsPsHy1o5XK7YBoNOIehhUZYTCakMz3dKOB5jfbEehWJE1AF+fd3Gw/MSZXzTbzIjY8YyhY79bp30tYh+3R5o3Bstndtj5C/4xwL4Wn7kzrDkP8M2jw1v/z7jVvs+mlWw72oygkZbIC459gJc3he+zyZ22pnssruTqp8OfuW/NtKNxLrt/qFk3KKJHbwSmMbHTfsWLIvd0LviMPqEXEghTgCeBrs1Uf1XVk/K0zXEcp1fUb3RlYSQQAJ5X1fDU2XEcp5+pZ2dsISQQslIugbB4nf1z0XEcJ3dKFRwFozASCMBUEXlQRO4VkYNDTyiXQJgwbGIOXTqO42RDS5r5KBo96tGLyCnAeFU9u6xsBTBBVdtFZCCwWFXHlq3RX9HDOX8HnFG2Rj8IGK6qK0VkP+BWYC9VNbN5jxq+i2l4NRr7McmCDR1hp9vE4bZswrK21WZdk9FXTOagMyKUdOL4t4bbRNyWMcmCGGsIS098e56tgmElLwdbM71at9omYzq1IXLGIaYVccfvckOh47vvWGe2OeNeW26hmi38MX324cbr2hCZcsYsiGnpP1MKBybM2XWt2eaS5ydEegsTs31o5H28Igc9+pX/+vbMA8uYX9wb7U9EDgO+QfIRuF5VL+lWPxmYQ7LnaBXwUVVtFZF9gG+SpF3tBC5S1R/3ZE8hJBBUdaOqrkwf3w88D+zWv1Y5jpMFa5BvOHJauhGRZuBa4H3ANOBYEZnW7WlXAN9LV03OB7rC3dqAj6vqXsBhwH+JyOieTC+KBMK49MUjIjsDuwL/6M05Hcdx8kRL2Y8e2B94TlX/oaqbgB8BR3Z7zjSSsRfgnq56VX1GVZ9NH78ELCOZ9UfpcaBX1ceBLgmEh4GrSCQQPikijwAfI1m37xEROVpEWoEDSSQQ7kir3gY8IiIPAbeQyBTbuf0cx3FqTQUz+vLAkfQ4oexME4FFZX+3pmXlPMxrCsFHAyO6JttdpJn5WkhWQKIURQLhp8BPs9jiOI7TH2SYqb/2XNXZvJYetRrOAK4RkeOA3wMvUua6SjPx3QR8QrVny1wCwXEcJwOVDPQ98CKwY9nfk9Ky1/pKlmXeDyAiw4EPqOrq9O+RwK+As1X1r1k6rFsJhMnD7QQJUk30gtpRGYOawtvxY9E9u48Zb9YtaQ8HE8Uif2J9qRF5sUvJlhFYaWyPh3hiCkvOIBZZc8W8r5l15xgJUCaW7AiKGC80hSUVpuoQs80asV/vMkNeAuBAHRksP+9ee9v/VYevMeuu/lU4IieW4CZ2ldZK+L6YWKruY2+d7yCx9eived72E35pir0X5tvzw+HTgyND1iuR9zEPtDO3xCb3AbuKyFSSAf4Y4MPlTxCRscCqdLZ+FkkEDiLSQrIq8j1VvSVrh0WRQPgI8IWyp04HZqjqQ3na5ziOUy15zehVtUNETgbuIPmunpPKwZ8PzFPV24B3ABeLiJIs3XSNqR8i8WmOSZd1AI7raawshASCqn6fZOMVIvJG4FYf5B3HKRJqKa1Vcy7V24Hbu5WdV/b4FpLAlO7t/gf4n0r7K6IEwrEk4UYhOzZ7sle12flfHcdx8ibH8MqaUyQJhC7+HfhhqKJcAmHbodvn2KXjOE4cVcl8FI0sSzeHAD/pSu2nqqtE5EBei/G8CbD3v1eAiLwFaFPVx3p67uL1dpi95bwqRbaLxxxemzrD2/6bm+zvyUUsN+vGDwk73Za02a+pFHHGvnPwjsHyJU22gzm2lzu2Fd/Sj4+1sRyuABfOuyhYPmvmOWab2ITJcsS3iu1UjTnvB0XmQo83hSUQLOkBgHNuH2XWnTe9NVh+8aM7mG1iMhctxutaUuV+eEs24xltM9vsiJ2L4KoFtgTCGbuFr8Xlz9rXorlKWY+slDqKN4BnpWjhlcdgzOYdx3H6kyoktApDISQQ0vM2kXiUg+vzjuM4/YmWJPNRNHqc0adhP10SCJ3AgyQSCDeIyBeA5cAns3QmIkeTrO+PI5FAeEhV35tWvw1YpKquceM4TuEo4gCelUJIIKR1vwMOyGKP4zhOrannpZuirdE7juMUknqe0feYeKSqk9ZAAmHv7Q/I1fBY0o+BTeEoilgkTDWMb7EjMpZssrfOHzpkSrB8SmSr+8om2/aNkbiWHTvD1+LZ5nBkEsBOETtWGVIMs+ZdaLa5esZ5Zl1rU9iO2Lb/1RE5iFVGohWAnUstwfJFhg0AE9S2w3pHTv7nlWaba381xqzbZJxxWCT8L3ZHtxkSCDEZho2RsXFgFR+fz+4TjsYBuOphO+vchQt+0OtR+rlp781s8RueuKNQ3wpFkUBoIdktO5Mkeu5z6VKO4zhOISgVMD4+K4WQQAA+DaCqbxSR7YBfi8ibs8hvOo7j1IIiboTKSlEkEDZnU1HVZcBqktl9dztek0BYvyxLl47jOLlQz+GVRZFAeBg4QkQGpNKd+/F6vWagmwTCEFum2HEcJ29Usx9FoygSCHNI1u3nAS+QbMiy9+/3AZame4ymiH58zFFrtVvebidZjjlqY9vgq2nTHqnL+x621uZiDtdTHzjfrDtt5lkV2xB7TbFrUQ2xmZV1w1s69QCnRhy1l99utKvyJVkKAAOjSxrV3UvWGec8GJb7ADhpwkuRM/aeIs7Us1KI8EpV7QBO6/pbRP4MPNN/FjmO47yezlKVIkEFoBASCCIyVESGpY8PBTpU9YnenNNxHCdPGnrppkYSCNsBd4hIiSS11seqejWO4zh9RMOHV/a1BIKqLgB2z2KL4zhOf1DP4ZV9sjO2FkweM900fOm61cHymMM1pkneWQq7yWLJvKN1Rl8x+0ole0vBio/sadade09452Rsg0JsPW+90TKmR2/pmIN9LWI2xBykX593sVn3/2Z+MVge0zGPOa3bjITyQ8XeKxrrqxo99di1uHBKOCfCVxdUF7E2MPKuLDI06XcQOyl77D22rkVs13bsHrx6wY97PUo/sOORmQfLGYt+XqhvhT5xxtZCAsHpGWuQ3xqxBnmn91iDfKNRz87YWksgXA78K7AJeB74pKquTh29twBvBm5U1ZPztMtxHKe31PMafa2/ou4C9k43Wj0DdAU9bwDOBc6osT2O4ziZ0AqOolFrCYQ705h5gL8Ck9Lydar6R5IBP2bHZgmEtRvs/KqO4zh5U1LJfBSN/pRAOB74dSUNyiUQhg+2dws6juPkjapkPopGv0ggiMjZQAfJl0RVjBo4zKzbY1xYl3pZ+ytmmw2ldrNu+IBwJvsOIxqnWgYYuvc9cfo9o4PlXz/M1rD/6m+2MevWRdQnZnYMCpb/dcB6s81UtSMvWiV83SeWBpptYlhO12/Os2/Rs2aebda9pPaPzINK4XvwgWa7zRQNXz+wI2hid0Usoum/FuwQLL9w3xfNNnPutyUGVhq6/W/XkWabRWLfSzHbrRloTM9/WaSvPKhnKd2aSyCIyHHAvwDv0nqN7XQcZ6ujs4Az9azUVAJBRA4DvggcobqVxGQ5jtMQlJDMR9GoqQQCcA0wCLgr3VD0V1U9CUBEFgAjgRYROQp4j+vdOI5TFLSAA3hWai2B8IZI3ZQstjiO4/QH9bxGX7cSCONH72kabm2r325w2GkJsGR95eGaMZmDaLsqJBBi79MHtplu1m1rfJd/8V3h7fEA59xtRzQNMVb7Xo04cLeJzCfWGh+fEZFVxdgdu4Kwc3dExIaL511k1p0Y2VE73HCTxmQJhkVdq7UhZsFpOy02665YOMGse9VIoh677jGsdz/2iYtd96sW/KjX0/E7tz8m82D5nqW97y9P+mRnLDCVgAQCScrA0M7Y/YHZXacBZqUCaE4vsAZ5x8kTa5BvNOr5VfbJSBCRQHgPcJaqdojIpSQ7Y79EkjR8Zlo+AXhYRH5RtrnKcRynX6nnNfqi7IxtKysfTDF3ETuOsxVTkuxH0SjMzlgReYuIPA48CpwUms2XSyC0bQpLETuO4/QFeYZXishhIvK0iDwnImcG6ncSkXtE5MF0Qn14Wj5QROaKyKMi8qSIZEqSnGVGv8XOWOBA4Adp/U3AQVk6K3sRW+yMVdW/qepeJAqWZ4nIFttRyyUQhrbYjlXHcZy8yUvUTESagWuB95H4LY8VkWndnnYOcLOq7kuyZ+m6tPyDwCBVfSOwH3CiiEzpyfbC7YxV1SdFZC2wNzDPOs/qDesq7nvVhlfNuonDx5p1C19ZFiyPfW/HInKaJPz9WlI7gKsUi44y1AyWGhEoAF+625ZAuPTQl826z9w5PFg+QiJb0yN2DDLmGqsirq9YdIUVuRSTMohF1nw7Ip3wbzNODZaPaQpLZgC0RaKTLEmAWFhfe+Se2UbCMhJrIzZ8eaH9Obho0pJg+XEv2K93l+YRZl0MK4mIlfgG7HspL3IMr9wfeE5V/wEgIj8CjgTK9w0pyb4igFHAS2Xlw0RkADCEJLDF1nZJKcTOWBGZmhqOiEwG9gAWZDmn4zhOLegUyXyULzOnxwllp5oILCr7uzUtK2cW8FERaQVuJ9mkCknejnXAYmAhcEW6yhKlKDtjDwLOFJF2ki/Oz3QtFTmO4xSBSmb0qjqb10LGq+FYkiRMV6YikjeJyN4kvwY6gR1Ifsv/QUR+2/XrwKIQO2NV9SaStX7HcZxCkmM0zYtAuUzopLSsnE8BhwGo6l9Sn+VY4MPAb1S1HVgmIn8CZgKmDUejAAAdaElEQVTRgb5+kyA6juPUkByjbu4Ddk2XrFtIlsFv6/achcC7AERkT5Kw8+Vp+SFp+TDgAOCpnjqs2+Tg+47ZxaxrMi60VQ6wvrTJrDt4h52D5S902HrvMToNx+qQpuo02Fdq2Pa3athxCvB0k/16T7nLdqB98+hwu0/+zHaeHhjRK3+8KewknVayHXwxHpCwk97Sjgd4qtl2FlsOV4BbHghHFX9iv9PNNjPUtuOVpvB9MSjihx8ckc59UcLvyZsi1za2Q/HK1vHB8jc2w3mHrgzWnXuXPZccHPk8Wu7iyZE8BS802e9jHuS1uSfdGHoycAeJIsWcdIn8fGCeqt4GnA58R0ROS7s+TlU1HVtvSEPRBbhBVR/pqc+iJAefAjwJPJ0+dbOqpeM4xcYa5BuNPDdCqertJE7W8rLzyh4/wZaTZVR1LUmIZUUUJTk4wPOquk96+CDvOE6h6KzgKBqFkEDISnnI0tJ1L/XcwHEcJydcAiGf5OBT0+2+94rIwaEG5Ttjtx8WzofpOI7TF5QqOIpGUZKDLwZ2UtWVIrIfcKuI7KWqPe74chzHqQVFHMCzUggJBFXdCGxMH98vIs8DuxGRQHjk5fl9b2zKo6UFwfLxw2wZgRXrK/+OiskcxBKPfHL8AcHy58WOQrC2mEM8WcQZt4Z/BH73HfbmvPPuHWrWWck7FjVVp1A91DjfA822BMLAyA/bmJyBFV0z9/4rzTanztxCv2oz1jsSzUkdqWs2Kp+KRFxZbcD++X/cnfY1um5yWDYB4OKF4SieGPMjkTUx2/OgjnODF0YCYVwq9IOI7AzsSg8bABzHcWpJRwVH0SiKBMLbgPPLJBBOyqLf4DiOUyvqOUlGUSQQfgr8NIstjuM4/UERo2my4klFHcdxMuDO2G7UQgJhx+Hj7P6rcMp0qL3NYUhzS7C8M6IFvvO225t1y9rDjtpmQ6ce4o7akvGjcpKG7QZYIfbrbYvc0tNKg4LlZ9y7rdnmqsNtqYhzbh8VLJ+g9q0Zcyw9b7yuKRq2G2Bl9FrYdZacQczhevW8S8y6C2aeGywfEFkzsDTsAVYbr2u0hh3WPfGKhO+LN2NLXHx9oS358MUJS8266xaHHbXD1X73X468j3ngA31KLyQQPgJ8oeyp04EZqvpQnvY5juNUSz2v0RdCAkFVv98lfwB8DJjvg7zjOEWiQ7IfRaOIEgjHAj8y7NgsgbB6/fIsXTqO4+RCXjlj+4MiSSB08e/AD0MNyiUQRg+x1+gdx3HypoRmPopGlhn9FhIIwIHAD9L6m0hSAWYmIIHQVf4WoE1VH6vkfI7jOH1No2vd5EpIAqGMYzBm893Z2Glv45ZI9IpFZyTqJhZdY/FCp53ydruWcKTJ8k22bIJGZgnW1u9qb7hYJEebURWTVLj6V3ZEznnTW4Plsx/ZMVgOcRlY61q0VznLil0LK1FIbInWiqwBOHfeBcHyi/az28Rel5VopyPSppqrZEX39IQVWQNw8qTFwfLrF9lihn0ugdCnZ+9bCiGBkNY1AR/CWJ93HMfpTxp6Rl8jCQRIZBAW9ZTN3HEcpz/okPqd0xdCAiGt+x1JolvHcZzCUb/DvEsgOI7jZKKISzJZqVsJhAFNlZsekxGwHFcAAyS8ZTzmpG0S+3wr218Nlo8fNNpss2yTLSOQN7EbusW6hBE/WOzaXvxo2Ln2lX+2E07HnLsW1W36j1+LQca1iOmWx+QMLKfr2feHnbQAF0cctRYxx1zs9VrtBkVkCdYbsgk9cW3rhGD5aXuHnfcAlz7et1nnihg2mZWiSCC0AN8GZpLca59Ll3Icx3EKQf0O8wWRQAA+DaCqbwQOBa6UamIkHcdx+ogONPNRNIoigTCNJIwTVV0GrCaZ3Xe3Y7MEwpoNdpy64zhO3rgEQu8lEB4GjhCRASIyFdgP2GLHTLkEwqjBY6vo0nEcpzoaOo6egASCiBwIvD+tvwm4rJJOAxIIc4A9SZKBv0CyISu63S6m3W63sevaI+9Oi+H4je1WjTl+LdZ0tJl1Ow0aY9ZZuyOfl43sYejHx7LltEduVeuGif1cjTlCO4121/7Kfr2nRhy1n799eLA8tsM1RnvE4T7Y8rpGuorZYb2PMYfrWRFH7TkzzzZsiOQ9MGtgrVHbIco2hsb9+sj5qmH2Y/aO6RNH9K3QYezzXnQKIYGQLuecVvacP5Os4Tu9wBrkHSdPrEG+0SjiTD0rhZBAEJGhIjIsfXwo0KGqT2R+FY7jOH1MPatXFkUCYTvgDhEpAS+SJB9xHMcpDNYyYz1QCAkEVV0A7J7FFsdxnP6gnpduXALBcRwnA/XsjJUtJeFzOGkNJBD+aeIhpuEvbbCjMqqhmjW32LZ/ay+YRiI8YnIL9+8XlgS49plQpsaEaremW1Ej6yLznRGRuJuNRrthkW31sZycn53yYrD8vxbY2+Njmu6bohFIYUM2Rc43LOIWi90zFrF788J5FwXLY1E81UggrJAOowZGR+aShpw/YEdqxT6Jsfvi/AXf77VY/fFT/i3zQDBnwS2FyhxbawmEC4AjSe6nZcBxqvqSiOwB3ADMAM5W1SvytMtxHKe31POMvtYyA5er6nRV3Qf4JXBeWr4KOBXwAd5xnEKS54YpETlMRJ4WkedE5MxA/U4ico+IPJgqEBweqF8rImdksb3WEgjlufKGkf4SU9Vlqnof0N6DHZslEJaseylLl47jOLnQqZr5iCEizcC1wPtI5F+OFZFp3Z52DnCzqu5LEsp+Xbf6q3hNWaBHely6KZNAeKuqrkjj6OeSSCDMFZHjSSQQjsrSYRqq+XFgDfDOrIZCIoEAzIb4Gr3jOE7e5Bgfvz/wXFc2PRH5EcmSdvneIQVGpo9HAZtntiJyFDAfWJe1w5pLIKjq2cDZInIWcDLwlaxty2nr3GjW7TE07Hhb2hHWgQfYVLIdSkOaW4Ll1SQNjxGTdYg5zb/6bDjJ8gX7hx2TAJfdZzsn2yI/Pid3ht1kjzbb129iyb7NlhgveVhM1D3yefvqgu2C5Rfua1+LSx+0r8XaiBLHm0qDg+VPNdmJ60dHdpFaMhKxn90xOQPL6RqTTbhmxnlm3atGKr2xOoARxvu1sqlyZ3aM2H2xpo9T/VWyRi8iJwAnlBXNTieqABOBRWV1rcBbup1iFnCniJxCsvrx7vS8w4Evkaj8Zlq2gdqv0ZfzfeAD/di/4zg5YA3yjUYla/TlAozpMds4rcWxwI2qOgk4HLgplW6fBXxdVddWcrJaSyDsWvbnkcBT2U11HMfpP3KUQHiR16vzTkrLyvkUcDOAqv4FGAyMJZn5XyYiC4D/BL4sIif31GGtJRAuEZHdSb70XgBOAhCR8STKlSOBkoj8JzCtm/PWcRyn38hRAuE+YNdUkv1Fkknzh7s9ZyHwLuBGEdmTZKBfrqoHdz2hLJz9mp46rLUEQnCpRlWX8FoSEsdxnMKR1+ZSVe1IZ+F3kOwNm5NOqM8H5qnqbcDpwHdE5DQSr9Rx2gsDXALBcRwnA3mqUqrq7cDt3crOK3v8BFuqC3Q/x6ys/fXJQF8LCYQYVnTN+IEjg+UArZteNuv6QiaiUmI3mVV349/sH0knT7T3IVyxOBy5kvQVJm93XN5XfM79dsKK06a0mnVfXmhnMrPijJqrvBrWa47FdlVTF4usOfmB8826i4wonliGoAEN5Kh1UbOUXkggHAlckJZ3AP+pqn/M0zbHcZze4BII2bEkEO4G3pSWHw9cX2O7HMdxotRz4pGiSCCsLXM0bC4P2LFZAmFF25IsXTqO4+RCXhII/UFhJBBE5GjgYpJsU/8calsugbDv+H8q3tV0HKdhqeelmx716NMtuONT6YKushXABFVtF5GBwGJVHVu2Rt+jCmUqgTBYVb/SrfxtwHmq+u5Y+13H7ZfrVZ/QMtqsa924KlguVTrdmqTydjG5hfcP3yNYPjKi6b4+sl389N1tuYCrnw47eGNSASMj2/43GnYMiTjxYrrjaww7hkZ+vMb06E+fZP9yvLI1LD0R07AfFtHmt4j97LZkEyCu6W8RcziebUgnnDHzy2abWOLwmHXWWxz70Mdsn/VC7/Xo373jezOPOb9ddEehvNCFk0BQ1d8DO4uIHe7gOI5TY1Q181E0CiGBICJvkDRbuIjMIEkgnm+aKMdxnF5Qz87YQkggkMzsPy4i7cB64N97swvMcRwnb/JWq60lRZFAuBS4NIstjuM4/UE9zzxdAsFxHCcDRVySyUrdSiCMbbHlDJoMn31sa3pbyU4W8Y7huwTL53esMdvEsH4CDmkKJziBuDPl2VJY8uFg7EiihWInbvnsU9uYdd+aGY7I+dh9w8w2B4l9vme0LVi+F0PNNgMjETmPET7f29W+X55osq/FcS+Ek4sA/OSwsBvpuDvtNm/GtmO1hCOGBkWiZ0ZE7oyXJJyZc2ppoNkmJmdgRddcMe9rZpvPzdwiHepmBkdst+zYMZLE5tnI+5gHPtCn9EIC4R3Az0nSYwH8r6raohuO4zg1pp7dhrVeurlcVc8FEJFTSSQQuhyyf1DVf6mxPY7jOJmo5xl9ISQQslIugbB0na2+6DiOkzclLWU+ikaPA32ZBMIhqvom4HPAf5NIIEwn2fh0ddYOReQiEVlEEn9frpd6YPpF8uu0zy0oz8O4/TA7obPjOE7e1HMcfSEkEERkJFBS1bUicjiJ43bXWPudx+6b69XUyLew5TzdYfAYs82yTZU7amM2xG6eD4yYFiyPbfuPOd1iKdMsh/Zpu9iyCdc8b+vibzQ2rg+qctP2KkMlPub4i9EWuVIDjXOetZMtm/D1hROqsqMarGsYu5eq0Y9fZjh9Ab4x7xKz7mJD3z5GTP6iKTIi5CGBUIm+1oNL/uQSCCmbJRBU9ZWurOZp5pWBLoHgOE6RqOcZfVEkEMaXSSDsn9rlEgiO4xQGreBf0SiKBMK/Af9PRDpIJBCOcQkEx3GKRKmOh6SiSCBcA1yTxRbHcZz+oOG1bopINVrwMR34UuR8TRLW1F6+6ZVgOcCEQfZuUNNRKxHnaRU3WRslUwu+M6JHH2OQ4ay75HnbyfilKbaj9qoF4XYDI+bFLG8y3saBkfc3pkcfY7BxzosXhnXqAb44YalZd91iu101WM7JATmnch+vA81dvTGH61mGvj3AJUa72H0RCzDIgyIuyWSlbiUQnJ6JJfxwnLywBvlGo+GXbrLSCwmEL5A4dbts2hMYp6rh1E6O4zg1pp5n9LUOr7xcVaer6j7AL0k3TKnq5aq6T1p+FnCvD/KO4xSJkmrmo2gUUQLhWOCHhh2bJRBe2bAiS5eO4zi5UNLOzEfR6HHppkwC4a2quiKNo59LIoEwV0SOJ5FAOCpLh2mo5seBNcA7u9UNBQ4DTg61VdXZwGyAXcbOKN7XpuM4DUsRN0JlJcsa/SHAT1R1BYCqrhKRA4H3p/U3AZdl7TCVUjg7lUA4GfhKWfW/An/KsmwTi6DJG2ttLhb5s2TjarNuByMi56WNL1dmWIqtvx9rEyMSnVTFZf/2/Ilm3Rm7tQbLv/WMLZsQM8GSaIi93lhdLEKlmnlbLLLm5EmLg+XXtlYnm1ArV3y181crsgbgTCMiJ9amr19vPW/tKYQEQhnHYCzbOI7j9CcugdBLCYS0bhTwdpIEJI7jOIVCVTMfRaMoEggARwN3quq6Sl6A4zhOLShiNE1WCiGBkNbdCNyYxR7HcZxaU8SEIlmpWwmEIc12Iu1qaC/ZLqWWpvBlqnYDxZrOcALrXYZsZ7ZZ2m7LLWwwNN03SMlMBL1W7Js2JgkwsTN8LVY02+cbHLnNLn82nEDmzH3CTlqAOQ/uaNZtNF7XBLVtWBnZ2bneuLYAk41rO7/J1mcfHkn0ff2i8LU4bW/7Wsx+zL4W1vs4rArNeYB1hmzGGG1mqHHOVyIi8TE5A8vpajlpAS6YWbm+fSXkufYuIocB3yDxIV+vqpd0q9+JZHI9On3OmamEe1cuj0+R+MFPVdU7euqvT3bGAlMJSCAAOxPeGbsNMAfYBdgAHK+qj+Vp29aINcg7Tp5Yg3yjkdfau4g0A9cChwKtwH0icpuqPlH2tHOAm1X1myIyDbgdmJI+PgbYC9gB+K2I7KYaD97vkxl9RAJhpJEc/MvAQ6p6tIjsQXIR3tUXtjmO41RDjmv0+wPPqeo/AETkRyQT4PKBXoGR6eNRQFeS7COBH6nqRmC+iDyXnu8vsQ6LsjN2Gkl0D6r6FMk31/ZZzuk4jlMLKom6Kd/Fnx4nlJ1qIrCo7O/WtKycWcBHRaSVZDZ/SgVtt6AoO2MfJtmA9Yc0w9RkYBKwtFvbE4ATACaOmMq2Q/27wHGc2lDJGn35Lv4qORa4UVWvTDeo3iQie1d7siwz+i12xgIHAj9I628CDsraoaqerao7kmyY6pI6uAQYLSIPkXxzPUhgw52qzlbVmao60wd5x3FqSWeplPnogReBci/6pLSsnE8BNwOo6l+AwcDYjG23QHpyMIjIKcD4VLqgq2wFMEFV20VkILBYVceWyRRf0WPHyXLP7aq6d7dyAeYD07st9byOyWOmV7xgFnutEpFU6DAicpojiUJiWH3F7NumZYRZd+jgycHyoZHv8Y2RaJLYhbWuUswTZCXoiLWLbWc/afslZt1lS8cFy1si1yI2U9sUqbPOGJPGGBSps+QbOiM2nDjcVguZu3asWVcNVgDNYtlktpmg1UXHWe9/e8Tve+48OyJn4Nide+0xHjZ0SuYxZ13bArM/ERkAPEPih3wRuA/4sKo+XvacXwM/VtUbRWRP4G6SJZppJJPs/UmcsXcDu/bkjC3EzlgRGS0iXXfEfwC/jw3yjuM4tSYvmWJV7SBZzbgDeJIkuuZxETlfRI5In3Y68GkReZhEFuY4TXicZKb/BPAb4LM9DfJQnJ2xewJzRUSBx0l+tjiO4xSGPKUN0pj427uVnVf2+Am2DFHvqrsIuKiS/gqxMzZdg9otiy2O4zj9QT1nmKrbnbGO4zi1pNSzk7WweHJwx3GcDNTvfJ7KNgEU+QBOKGqbRu2r6Pb5tfBr4Udy9Gfikbw5oeen9FubRu2r6PbVsq+i21fLvopu31ZHIw30juM4TgAf6B3HcRqcRhroq9GVqFWbRu2r6PbVsq+i21fLvopu31ZHjxIIjuM4Tn3TSDN6x3EcJ4AP9I7jOA2OD/SO4zgNjg/0juM4DU7DDfSpjnOofLyIfFNErhWRMSIyS0QeFZGbRWSC0eawssejROS7afrEH8RSHYrIAyJyjojsUoHdw1OZ0sdFZI2ILBeRv4rIcVnPETjnHpG6JpFEUF9EWkRkRipBXYm9M0RkdA/Pm57d4i3azhSRo0XkiNhryXguvxavnWt4Bc/NfB0qadPTteqh7TgR2VdEplfyWrZq+ntrbjUHMMM49iNJghJq8xsSeeUzgUeAL5FkajkF+LnR5oGyx9cDF5KkOTwNuDVi33zgCmAh8Pf0+Tv08Jp+DhxHkjHm88C5wK4kqqFfq/I6LTTKjyJJ07iYJC/A30gSGLQC/2q0ua7s8UHpa7uHJH/l4REbOoFngQuAaRntfjswD/gt8DLwS+BPwO+AHf1a9Nm1+CcSffTHgbcAdwHPp6/rQKPNOWWPp5Ek1JgPLADeErGhI31NnwJGZ7R7WtrmOWBT+l7NB24ERlVzLbaWo98NqMro5APzf+mHq/ux3mjzYNnjhd3qHjLaPGA9x2oTaHcwcB2wJLUvqM0BPNzt7/vS/5uApyJ9XW0c/w28Yl0LYDwwFXgF2D0tnwzMy/Ca7gFmpI93ttqU9bU3iX72cyT5gc8EpvTQZlz6eCrws/TxocCdfi0yXYvPG8fpwCqjzd+BN5KkCl0BHJSWzwD+lOFa/Ap4X/p4f+DPEfseBf6FJKXoSpKJzjHAkEibv5a9P/uT5K0G+DRwi9XOj/rVunkSOFFV39n9ILlBQ5S/1u9F6srZTkQ+LyKnAyPl9TkAM107Vf2Dqn6GJA3YpSQfohDrROQggDTLzKq0fQk7gx8kSV8eA+7vdswjmfVYdi1R1fkkX3pPp2UvZHxdI1X1gbTNP3poo6r6mCa5gt9A8qHcDvijiPzZaNOsqsvTxwtJBl1U9S7iGe/9WrzG14BtgBHdjuERGweq6qOa5IdYrqp/TPt6ABgS6auLHVT112mbv/fQpl1Vf6mqHyH5Fft94ENAq4j8wGgzpOz96fpSQlW/A+yVwb6tlnrVo5+FfbOeYpT/XESGq+paVT2nq1BE3kDyczPEd0g+HJAsoYwFlovIeOChiH1bnE+TdF+/SY8QJwHXp+kWHweOT+0bB1wb6es+4DFV3WKgSHP4BhGRpvRL5PiysmbASvK5h4g8QvKlM0VEtlHVl9O17Vhi0Nd9SaUf0L+nX55vM9rME5HvkvxqO4JkmQIRGUo8naxfi9d4gGR58f4tjBD5D6NN+WfqrG511uvaWURuI3ltk0RkqKq2pXUDI/Ztvhaqup4kPd7NIjKKZDktxPMici7JtXg/6WcwzVtdr5PW2tDfPyn68gA+UdQ2efUFbAsMrfAcbwYGB8qnAB812kzudgxMy8cC74/09eEqXuNA4DPANSSz3ua0fAgwOdLOr8Vr7XYnXfIJ1G1vlB8Run7ALsAXjTZv73YM7+qDJJ+pZd8ZVVyL0cBlJH6Ki4ARafko4IBKz7c1HQ0tgSAiD6jqjCK2qXVfjuNsvTT6z53Y2nZ/t+nzvqxQ0yK06Yu+RGSkiFwsIjeJyIe71V3Xn236oa9qwolr0qaXfV1XaV9O4w/01fxcqVWbXPpK47dDx37APqET1KpNrfsCbiD5IvwpcIyI/FREBqV1B/Rzm1r3dSPwBElo5D3AeuBw4A/At/q5TW/6erKKvrZ6Gn3p5kFV3beIbfLqS0Q6gXsJz/QPUNUtIh9q1aYf+npIVfcp+/tskoHgCOCu0JJXrdr0Q1+b7xMRWaiqO1nnrHWbWvfl1G/UTVb+VOA2efXVFWr6bPcnisgi4xy1alPrvgbJaxE0qOpFIvIi8HuSsML+bFPrvqoJJ65Vm1r3tdVTlwO9iHw+Vq+qV6X/n1zrNrXui+pCTWvVptZ9/QI4hGT3JACqeqOILCHZNNWfbWrdVzXhxLVqU+u+tnrqculGRL4Sq1fVr/ZXm1r3lRUR+YSqzi1im0btq+j21bKvotvX6NTlQO9Ujnioac37Krp9teyr6PY1OnW9riUik0TkZyKyLD1+KiKTitCm1n1lYKsONe2nvopuXy37Krp9DU1dD/QkoWe3ATukxy/SsiK0qXVfPdHwoaYF7Kvo9tWyr6Lb19DU+0A/TlVvUNWO9LgRGFeQNrXuqyd85lb7vopuXy37Krp9DU29D/QrReSjItKcHh8lkTwtQpta99UTW0OoadH6Krp9teyr6PY1NHXtjBWRySQhZgeS/Fz7M3CKqpqx1rVqU6u+soZl9kebRu2r6PbVsq+i2+ck1GUcfRnnk6g5vgwgSQqzKyiTm+3HNrXqa4RRHqNWbRq1r6LbV8u+im6fQ/3P6LeQEAiV9UebWvflOI5jUe9r9E0isk3XH+nst6dfKbVqU9O+PNS09n0V3b5a9lV0+7Z6tACi+NUewMeBp0iSLV+QPv5YEdr0Q193kaTSG5Aex5GIXvV7m0btq+j2+bXwY/M1628Dev0CkszwJ6fHtCK1qbF9WyQrD5X1R5tG7avo9vm18KPrqHdnLKr6BImudeHa1LivlZKEYv4w/ftYMoZy1qBNo/ZVdPtq2VfR7du66e9vGj/yOUhyl94GLAeWAbcCOxahTaP2VXT7/Fr4sfma9bcBfuT0RsJcYJuyv7cF5hShTaP2VXT7/Fr40XXUe9SN8xrTNY29B1DVVUBPIZm1atOofRXdvlr2VXT7tmp8oG8cPNS09n0V3b5a9lV0+7Zq/OI0DlcCfxGRn6R/fxC4qCBtGrWvottXy76Kbt9WTV3vjHVej4hMI0k7B/B/mkTvFKJNo/ZVdPtq2VfR7dua8YHecRynwfE1esdxnAbHB3rHcZwGxwd6x3GcBscHesdxnAbn/wMFLvHR7VBxgQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "corr = final_test.corr(method='kendall')\n",
    "sns.heatmap(corr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = corr.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = []\n",
    "for col in weights.index:\n",
    "    we = 1 / (weights[col] ** 3)\n",
    "    w.append(we)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.target = np.average(final_test, weights=w, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.044391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.129663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.151100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.175111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.053089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>test_5</td>\n",
       "      <td>0.000848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>test_6</td>\n",
       "      <td>0.003232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>test_7</td>\n",
       "      <td>0.069774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>test_8</td>\n",
       "      <td>0.000885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>test_9</td>\n",
       "      <td>0.003975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>test_10</td>\n",
       "      <td>0.082121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>test_11</td>\n",
       "      <td>0.018555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>test_12</td>\n",
       "      <td>0.023832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>test_13</td>\n",
       "      <td>0.030905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>test_14</td>\n",
       "      <td>0.003415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>test_15</td>\n",
       "      <td>0.014868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>test_16</td>\n",
       "      <td>0.374879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>test_17</td>\n",
       "      <td>0.019366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>test_18</td>\n",
       "      <td>0.072993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>test_19</td>\n",
       "      <td>0.004858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>test_20</td>\n",
       "      <td>0.380302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>test_21</td>\n",
       "      <td>0.040902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>test_22</td>\n",
       "      <td>0.004928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>test_23</td>\n",
       "      <td>0.014302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>test_24</td>\n",
       "      <td>0.003603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>test_25</td>\n",
       "      <td>0.078485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>test_26</td>\n",
       "      <td>0.066035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>test_27</td>\n",
       "      <td>0.006053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>test_28</td>\n",
       "      <td>0.308944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>test_29</td>\n",
       "      <td>0.031098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199970</th>\n",
       "      <td>test_199970</td>\n",
       "      <td>0.043051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199971</th>\n",
       "      <td>test_199971</td>\n",
       "      <td>0.177853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199972</th>\n",
       "      <td>test_199972</td>\n",
       "      <td>0.001985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199973</th>\n",
       "      <td>test_199973</td>\n",
       "      <td>0.023506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199974</th>\n",
       "      <td>test_199974</td>\n",
       "      <td>0.002584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199975</th>\n",
       "      <td>test_199975</td>\n",
       "      <td>0.053069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199976</th>\n",
       "      <td>test_199976</td>\n",
       "      <td>0.202876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199977</th>\n",
       "      <td>test_199977</td>\n",
       "      <td>0.184413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199978</th>\n",
       "      <td>test_199978</td>\n",
       "      <td>0.018290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199979</th>\n",
       "      <td>test_199979</td>\n",
       "      <td>0.143240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199980</th>\n",
       "      <td>test_199980</td>\n",
       "      <td>0.057953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199981</th>\n",
       "      <td>test_199981</td>\n",
       "      <td>0.012633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199982</th>\n",
       "      <td>test_199982</td>\n",
       "      <td>0.008142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199983</th>\n",
       "      <td>test_199983</td>\n",
       "      <td>0.008848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199984</th>\n",
       "      <td>test_199984</td>\n",
       "      <td>0.018226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199985</th>\n",
       "      <td>test_199985</td>\n",
       "      <td>0.075410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199986</th>\n",
       "      <td>test_199986</td>\n",
       "      <td>0.191851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199987</th>\n",
       "      <td>test_199987</td>\n",
       "      <td>0.023042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199988</th>\n",
       "      <td>test_199988</td>\n",
       "      <td>0.011697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199989</th>\n",
       "      <td>test_199989</td>\n",
       "      <td>0.034664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199990</th>\n",
       "      <td>test_199990</td>\n",
       "      <td>0.001904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199991</th>\n",
       "      <td>test_199991</td>\n",
       "      <td>0.002339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199992</th>\n",
       "      <td>test_199992</td>\n",
       "      <td>0.131813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199993</th>\n",
       "      <td>test_199993</td>\n",
       "      <td>0.004030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199994</th>\n",
       "      <td>test_199994</td>\n",
       "      <td>0.058654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199995</th>\n",
       "      <td>test_199995</td>\n",
       "      <td>0.024872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199996</th>\n",
       "      <td>test_199996</td>\n",
       "      <td>0.003751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199997</th>\n",
       "      <td>test_199997</td>\n",
       "      <td>0.001987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199998</th>\n",
       "      <td>test_199998</td>\n",
       "      <td>0.040447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199999</th>\n",
       "      <td>test_199999</td>\n",
       "      <td>0.028008</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID_code    target\n",
       "0            test_0  0.044391\n",
       "1            test_1  0.129663\n",
       "2            test_2  0.151100\n",
       "3            test_3  0.175111\n",
       "4            test_4  0.053089\n",
       "5            test_5  0.000848\n",
       "6            test_6  0.003232\n",
       "7            test_7  0.069774\n",
       "8            test_8  0.000885\n",
       "9            test_9  0.003975\n",
       "10          test_10  0.082121\n",
       "11          test_11  0.018555\n",
       "12          test_12  0.023832\n",
       "13          test_13  0.030905\n",
       "14          test_14  0.003415\n",
       "15          test_15  0.014868\n",
       "16          test_16  0.374879\n",
       "17          test_17  0.019366\n",
       "18          test_18  0.072993\n",
       "19          test_19  0.004858\n",
       "20          test_20  0.380302\n",
       "21          test_21  0.040902\n",
       "22          test_22  0.004928\n",
       "23          test_23  0.014302\n",
       "24          test_24  0.003603\n",
       "25          test_25  0.078485\n",
       "26          test_26  0.066035\n",
       "27          test_27  0.006053\n",
       "28          test_28  0.308944\n",
       "29          test_29  0.031098\n",
       "...             ...       ...\n",
       "199970  test_199970  0.043051\n",
       "199971  test_199971  0.177853\n",
       "199972  test_199972  0.001985\n",
       "199973  test_199973  0.023506\n",
       "199974  test_199974  0.002584\n",
       "199975  test_199975  0.053069\n",
       "199976  test_199976  0.202876\n",
       "199977  test_199977  0.184413\n",
       "199978  test_199978  0.018290\n",
       "199979  test_199979  0.143240\n",
       "199980  test_199980  0.057953\n",
       "199981  test_199981  0.012633\n",
       "199982  test_199982  0.008142\n",
       "199983  test_199983  0.008848\n",
       "199984  test_199984  0.018226\n",
       "199985  test_199985  0.075410\n",
       "199986  test_199986  0.191851\n",
       "199987  test_199987  0.023042\n",
       "199988  test_199988  0.011697\n",
       "199989  test_199989  0.034664\n",
       "199990  test_199990  0.001904\n",
       "199991  test_199991  0.002339\n",
       "199992  test_199992  0.131813\n",
       "199993  test_199993  0.004030\n",
       "199994  test_199994  0.058654\n",
       "199995  test_199995  0.024872\n",
       "199996  test_199996  0.003751\n",
       "199997  test_199997  0.001987\n",
       "199998  test_199998  0.040447\n",
       "199999  test_199999  0.028008\n",
       "\n",
       "[200000 rows x 2 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.to_csv('#a2.csv', index=False)\n",
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}