argsort feature augment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "import scikitplot as skplt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import lightgbm as lgb\n",
    "import pickle\n",
    "import gc\n",
    "from helpers import save_model, lgb_trainer, lgb_trainer_no_aug, test\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataframe(dataset):\n",
    "    return pd.read_csv(dataset)\n",
    "with multiprocessing.Pool() as pool:\n",
    "    train_df, test_df = pool.map(load_dataframe, ['data/train.csv', 'data/test.csv'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(models, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(models, handle)\n",
    "        \n",
    "def augment(x,y,t=2):\n",
    "    xs,xn = [],[]\n",
    "    for i in range(t):\n",
    "        mask = y>0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xs.append(x1)\n",
    "\n",
    "    for i in range(t//2):\n",
    "        mask = y==0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xn.append(x1)\n",
    "    xs = np.vstack(xs)\n",
    "    xn = np.vstack(xn)\n",
    "    ys = np.ones(xs.shape[0])\n",
    "    yn = np.zeros(xn.shape[0])\n",
    "    x = np.vstack([x,xs,xn])\n",
    "    y = np.concatenate([y,ys,yn])\n",
    "    return x,y\n",
    "\n",
    "def lgb_trainer(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in skf.split(X.values, y.values):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = augment(X_train.values, y_train.values)\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def lgb_trainer_no_aug(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in skf.split(X.values, y.values):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = X_train.values, y_train.values\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def test(X, y, models):\n",
    "    preds = pd.DataFrame({})\n",
    "    for i, model in enumerate(models):\n",
    "        preds[str(i)] = model.predict(X)\n",
    "        print(f\"Fold: {i} \\t Score: {roc_auc_score(y, preds[str(i)].values)}\")\n",
    "    averaged_preds = preds.mean(axis=1)\n",
    "    print(f\"Score: {roc_auc_score(y, averaged_preds)}\")\n",
    "    return averaged_preds, preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Augment rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/timetraveller/miniconda3/lib/python3.6/site-packages/pandas/core/frame.py:6211: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  sort=sort)\n"
     ]
    }
   ],
   "source": [
    "whole = train_df.append(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_ranks(df):\n",
    "    for column in df.columns:\n",
    "        if column in ['ID_code', 'target']:\n",
    "            continue\n",
    "        key = column + '_rank'\n",
    "        df[key] = np.argsort(df[column])\n",
    "    return df   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "whole = augment_ranks(whole)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_10</th>\n",
       "      <th>var_100</th>\n",
       "      <th>var_101</th>\n",
       "      <th>var_102</th>\n",
       "      <th>var_103</th>\n",
       "      <th>var_104</th>\n",
       "      <th>...</th>\n",
       "      <th>var_90_rank</th>\n",
       "      <th>var_91_rank</th>\n",
       "      <th>var_92_rank</th>\n",
       "      <th>var_93_rank</th>\n",
       "      <th>var_94_rank</th>\n",
       "      <th>var_95_rank</th>\n",
       "      <th>var_96_rank</th>\n",
       "      <th>var_97_rank</th>\n",
       "      <th>var_98_rank</th>\n",
       "      <th>var_99_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.9255</td>\n",
       "      <td>-6.7863</td>\n",
       "      <td>2.9252</td>\n",
       "      <td>9.4763</td>\n",
       "      <td>13.3102</td>\n",
       "      <td>26.5376</td>\n",
       "      <td>1.4403</td>\n",
       "      <td>14.7100</td>\n",
       "      <td>...</td>\n",
       "      <td>26612</td>\n",
       "      <td>107906</td>\n",
       "      <td>165151</td>\n",
       "      <td>144266</td>\n",
       "      <td>371497</td>\n",
       "      <td>83672</td>\n",
       "      <td>85889</td>\n",
       "      <td>367237</td>\n",
       "      <td>154549</td>\n",
       "      <td>220487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.5006</td>\n",
       "      <td>-4.1473</td>\n",
       "      <td>-0.4032</td>\n",
       "      <td>-13.6950</td>\n",
       "      <td>8.4068</td>\n",
       "      <td>35.4734</td>\n",
       "      <td>1.7093</td>\n",
       "      <td>15.1866</td>\n",
       "      <td>...</td>\n",
       "      <td>6534</td>\n",
       "      <td>59339</td>\n",
       "      <td>145133</td>\n",
       "      <td>102186</td>\n",
       "      <td>225371</td>\n",
       "      <td>91243</td>\n",
       "      <td>130521</td>\n",
       "      <td>361110</td>\n",
       "      <td>260836</td>\n",
       "      <td>238168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.6093</td>\n",
       "      <td>-2.7457</td>\n",
       "      <td>-0.3249</td>\n",
       "      <td>-0.3939</td>\n",
       "      <td>12.6317</td>\n",
       "      <td>14.8863</td>\n",
       "      <td>1.3854</td>\n",
       "      <td>15.0284</td>\n",
       "      <td>...</td>\n",
       "      <td>61117</td>\n",
       "      <td>155651</td>\n",
       "      <td>138496</td>\n",
       "      <td>51928</td>\n",
       "      <td>241797</td>\n",
       "      <td>191880</td>\n",
       "      <td>198398</td>\n",
       "      <td>143353</td>\n",
       "      <td>378978</td>\n",
       "      <td>355199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0604</td>\n",
       "      <td>-2.1518</td>\n",
       "      <td>2.3061</td>\n",
       "      <td>-19.8592</td>\n",
       "      <td>22.5316</td>\n",
       "      <td>18.6129</td>\n",
       "      <td>1.3512</td>\n",
       "      <td>9.3291</td>\n",
       "      <td>...</td>\n",
       "      <td>55152</td>\n",
       "      <td>92527</td>\n",
       "      <td>169836</td>\n",
       "      <td>390680</td>\n",
       "      <td>132410</td>\n",
       "      <td>232483</td>\n",
       "      <td>131761</td>\n",
       "      <td>198207</td>\n",
       "      <td>293197</td>\n",
       "      <td>72482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.8369</td>\n",
       "      <td>-1.4834</td>\n",
       "      <td>-9.4458</td>\n",
       "      <td>-22.9264</td>\n",
       "      <td>12.3562</td>\n",
       "      <td>17.3410</td>\n",
       "      <td>1.6940</td>\n",
       "      <td>7.1179</td>\n",
       "      <td>...</td>\n",
       "      <td>148063</td>\n",
       "      <td>86626</td>\n",
       "      <td>192857</td>\n",
       "      <td>208448</td>\n",
       "      <td>364423</td>\n",
       "      <td>129932</td>\n",
       "      <td>41602</td>\n",
       "      <td>148130</td>\n",
       "      <td>295811</td>\n",
       "      <td>92898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 402 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID_code  target    var_0   var_1  var_10  var_100  var_101  var_102  \\\n",
       "0  train_0     0.0   8.9255 -6.7863  2.9252   9.4763  13.3102  26.5376   \n",
       "1  train_1     0.0  11.5006 -4.1473 -0.4032 -13.6950   8.4068  35.4734   \n",
       "2  train_2     0.0   8.6093 -2.7457 -0.3249  -0.3939  12.6317  14.8863   \n",
       "3  train_3     0.0  11.0604 -2.1518  2.3061 -19.8592  22.5316  18.6129   \n",
       "4  train_4     0.0   9.8369 -1.4834 -9.4458 -22.9264  12.3562  17.3410   \n",
       "\n",
       "   var_103  var_104     ...       var_90_rank  var_91_rank  var_92_rank  \\\n",
       "0   1.4403  14.7100     ...             26612       107906       165151   \n",
       "1   1.7093  15.1866     ...              6534        59339       145133   \n",
       "2   1.3854  15.0284     ...             61117       155651       138496   \n",
       "3   1.3512   9.3291     ...             55152        92527       169836   \n",
       "4   1.6940   7.1179     ...            148063        86626       192857   \n",
       "\n",
       "   var_93_rank  var_94_rank  var_95_rank  var_96_rank  var_97_rank  \\\n",
       "0       144266       371497        83672        85889       367237   \n",
       "1       102186       225371        91243       130521       361110   \n",
       "2        51928       241797       191880       198398       143353   \n",
       "3       390680       132410       232483       131761       198207   \n",
       "4       208448       364423       129932        41602       148130   \n",
       "\n",
       "   var_98_rank  var_99_rank  \n",
       "0       154549       220487  \n",
       "1       260836       238168  \n",
       "2       378978       355199  \n",
       "3       293197        72482  \n",
       "4       295811        92898  \n",
       "\n",
       "[5 rows x 402 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whole.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf = whole[200000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf, testdf = whole[:200000], whole[200000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = tdf.target\n",
    "tdf = tdf.drop(['target', 'ID_code'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xval, ytr, yval  = train_test_split(tdf, y_train, test_size = 0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "    'bagging_freq': 5,\n",
    "    'bagging_fraction': 0.335,\n",
    "    'boost_from_average':'false',\n",
    "    'boost': 'gbdt',\n",
    "    'feature_fraction': 0.041,\n",
    "    'learning_rate': 0.001,\n",
    "    'max_depth': 4,\n",
    "    'metric':'auc',\n",
    "    'min_data_in_leaf': 100,\n",
    "    'min_sum_hessian_in_leaf': 10.0,\n",
    "    'num_leaves': 10,\n",
    "    'num_threads': 8,\n",
    "    'tree_learner': 'serial',\n",
    "    'objective': 'binary', \n",
    "    'verbosity': -1,\n",
    "    'seed':42,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.906994\tvalid_1's auc: 0.876464\n",
      "[10000]\ttraining's auc: 0.914307\tvalid_1's auc: 0.881109\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-e2cd42d44d3f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrank_models\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb_trainer_no_aug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXtr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mytr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_folds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-3-68d4ce82d6cc>\u001b[0m in \u001b[0;36mlgb_trainer_no_aug\u001b[0;34m(X, y, params, n_folds)\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0mtrn_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_tr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0mtest_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m             \u001b[0mmodel_lgb\u001b[0m     \u001b[0;34m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1000000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_sets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtrn_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_data\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mearly_stopping_rounds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m             \u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_lgb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m             \u001b[0mauc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_lgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/lightgbm/engine.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[1;32m    216\u001b[0m                                     evaluation_result_list=None))\n\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 218\u001b[0;31m         \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, train_set, fobj)\u001b[0m\n\u001b[1;32m   1800\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[1;32m   1801\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1802\u001b[0;31m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[1;32m   1803\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mFalse\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1804\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "rank_models = lgb_trainer_no_aug(Xtr, ytr, param, n_folds = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.8979343920178489\n",
      "Fold: 1 \t Score: 0.8989173868095288\n",
      "Score: 0.9009706772952017\n"
     ]
    }
   ],
   "source": [
    "q, w = test(Xval, yval, rank_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.8997925361059268\n",
      "Fold: 1 \t Score: 0.9014075481477227\n",
      "Fold: 2 \t Score: 0.9010311820861489\n",
      "Fold: 3 \t Score: 0.8997016035382924\n",
      "Fold: 4 \t Score: 0.9013254155059885\n",
      "Fold: 5 \t Score: 0.9010606279920454\n",
      "Fold: 6 \t Score: 0.9004653919785967\n",
      "Fold: 7 \t Score: 0.9014594451466209\n",
      "Fold: 8 \t Score: 0.9017516478143297\n",
      "Fold: 9 \t Score: 0.9008035122083742\n",
      "Score: 0.9019027132089482\n"
     ]
    }
   ],
   "source": [
    "q, w = test(Xval, yval, rank_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912828\tvalid_1's auc: 0.900054\n",
      "[10000]\ttraining's auc: 0.925068\tvalid_1's auc: 0.902264\n",
      "[15000]\ttraining's auc: 0.934735\tvalid_1's auc: 0.90222\n",
      "Early stopping, best iteration is:\n",
      "[11294]\ttraining's auc: 0.927695\tvalid_1's auc: 0.902475\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912384\tvalid_1's auc: 0.899655\n",
      "[10000]\ttraining's auc: 0.924822\tvalid_1's auc: 0.902414\n",
      "[15000]\ttraining's auc: 0.934502\tvalid_1's auc: 0.902163\n",
      "Early stopping, best iteration is:\n",
      "[11307]\ttraining's auc: 0.927468\tvalid_1's auc: 0.902561\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.913259\tvalid_1's auc: 0.894817\n",
      "[10000]\ttraining's auc: 0.925386\tvalid_1's auc: 0.897191\n",
      "Early stopping, best iteration is:\n",
      "[10958]\ttraining's auc: 0.927331\tvalid_1's auc: 0.897374\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.913119\tvalid_1's auc: 0.898361\n",
      "[10000]\ttraining's auc: 0.925291\tvalid_1's auc: 0.90206\n",
      "[15000]\ttraining's auc: 0.934923\tvalid_1's auc: 0.902597\n",
      "Early stopping, best iteration is:\n",
      "[14265]\ttraining's auc: 0.9336\tvalid_1's auc: 0.902635\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912763\tvalid_1's auc: 0.902202\n",
      "[10000]\ttraining's auc: 0.925142\tvalid_1's auc: 0.905548\n",
      "[15000]\ttraining's auc: 0.934764\tvalid_1's auc: 0.905535\n",
      "Early stopping, best iteration is:\n",
      "[12068]\ttraining's auc: 0.929336\tvalid_1's auc: 0.905847\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.913042\tvalid_1's auc: 0.8963\n",
      "[10000]\ttraining's auc: 0.925319\tvalid_1's auc: 0.899414\n",
      "[15000]\ttraining's auc: 0.934943\tvalid_1's auc: 0.899556\n",
      "Early stopping, best iteration is:\n",
      "[13022]\ttraining's auc: 0.93132\tvalid_1's auc: 0.899791\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.913637\tvalid_1's auc: 0.894207\n",
      "[10000]\ttraining's auc: 0.925838\tvalid_1's auc: 0.898188\n",
      "[15000]\ttraining's auc: 0.935442\tvalid_1's auc: 0.898521\n",
      "Early stopping, best iteration is:\n",
      "[14097]\ttraining's auc: 0.933804\tvalid_1's auc: 0.898616\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.913727\tvalid_1's auc: 0.894555\n",
      "[10000]\ttraining's auc: 0.9259\tvalid_1's auc: 0.897638\n",
      "[15000]\ttraining's auc: 0.935479\tvalid_1's auc: 0.897891\n",
      "Early stopping, best iteration is:\n",
      "[12812]\ttraining's auc: 0.931481\tvalid_1's auc: 0.898042\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912838\tvalid_1's auc: 0.892441\n",
      "[10000]\ttraining's auc: 0.925037\tvalid_1's auc: 0.896511\n",
      "[15000]\ttraining's auc: 0.934664\tvalid_1's auc: 0.897101\n",
      "Early stopping, best iteration is:\n",
      "[13110]\ttraining's auc: 0.931202\tvalid_1's auc: 0.897293\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912951\tvalid_1's auc: 0.899345\n",
      "[10000]\ttraining's auc: 0.92527\tvalid_1's auc: 0.900893\n",
      "Early stopping, best iteration is:\n",
      "[8824]\ttraining's auc: 0.922729\tvalid_1's auc: 0.900979\n"
     ]
    }
   ],
   "source": [
    "rank_models = lgb_trainer(Xtr, ytr, param, n_folds = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.9012618980921195\n",
      "Fold: 1 \t Score: 0.9028719460731515\n",
      "Fold: 2 \t Score: 0.9033342806415959\n",
      "Fold: 3 \t Score: 0.9019429897928757\n",
      "Fold: 4 \t Score: 0.9008802295110931\n",
      "Fold: 5 \t Score: 0.9020246711563589\n",
      "Fold: 6 \t Score: 0.9020896552245443\n",
      "Fold: 7 \t Score: 0.9018609699707041\n",
      "Fold: 8 \t Score: 0.9025413847540833\n",
      "Fold: 9 \t Score: 0.9009602185811338\n",
      "Score: 0.9028675461102015\n"
     ]
    }
   ],
   "source": [
    "q, w = test(Xval, yval, rank_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 400)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xval.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_10</th>\n",
       "      <th>var_100</th>\n",
       "      <th>var_101</th>\n",
       "      <th>var_102</th>\n",
       "      <th>var_103</th>\n",
       "      <th>var_104</th>\n",
       "      <th>...</th>\n",
       "      <th>var_90_rank</th>\n",
       "      <th>var_91_rank</th>\n",
       "      <th>var_92_rank</th>\n",
       "      <th>var_93_rank</th>\n",
       "      <th>var_94_rank</th>\n",
       "      <th>var_95_rank</th>\n",
       "      <th>var_96_rank</th>\n",
       "      <th>var_97_rank</th>\n",
       "      <th>var_98_rank</th>\n",
       "      <th>var_99_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.0656</td>\n",
       "      <td>7.7798</td>\n",
       "      <td>-2.0248</td>\n",
       "      <td>-9.2198</td>\n",
       "      <td>17.3089</td>\n",
       "      <td>30.9548</td>\n",
       "      <td>1.4918</td>\n",
       "      <td>12.8721</td>\n",
       "      <td>...</td>\n",
       "      <td>283352</td>\n",
       "      <td>7357</td>\n",
       "      <td>136022</td>\n",
       "      <td>153965</td>\n",
       "      <td>202685</td>\n",
       "      <td>380958</td>\n",
       "      <td>384304</td>\n",
       "      <td>14213</td>\n",
       "      <td>247572</td>\n",
       "      <td>368780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.5304</td>\n",
       "      <td>1.2543</td>\n",
       "      <td>-1.3809</td>\n",
       "      <td>-1.7257</td>\n",
       "      <td>15.4712</td>\n",
       "      <td>35.6020</td>\n",
       "      <td>1.6570</td>\n",
       "      <td>13.0783</td>\n",
       "      <td>...</td>\n",
       "      <td>57818</td>\n",
       "      <td>384042</td>\n",
       "      <td>254703</td>\n",
       "      <td>226045</td>\n",
       "      <td>324501</td>\n",
       "      <td>41135</td>\n",
       "      <td>14516</td>\n",
       "      <td>332780</td>\n",
       "      <td>96953</td>\n",
       "      <td>387184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.4827</td>\n",
       "      <td>-10.3581</td>\n",
       "      <td>-4.7057</td>\n",
       "      <td>-3.5065</td>\n",
       "      <td>14.1663</td>\n",
       "      <td>28.0256</td>\n",
       "      <td>1.3935</td>\n",
       "      <td>10.8257</td>\n",
       "      <td>...</td>\n",
       "      <td>79010</td>\n",
       "      <td>34644</td>\n",
       "      <td>232915</td>\n",
       "      <td>316034</td>\n",
       "      <td>380224</td>\n",
       "      <td>127703</td>\n",
       "      <td>379528</td>\n",
       "      <td>307586</td>\n",
       "      <td>137103</td>\n",
       "      <td>347787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.5374</td>\n",
       "      <td>-1.3222</td>\n",
       "      <td>0.0095</td>\n",
       "      <td>1.7021</td>\n",
       "      <td>2.5363</td>\n",
       "      <td>3.8763</td>\n",
       "      <td>1.5173</td>\n",
       "      <td>13.4083</td>\n",
       "      <td>...</td>\n",
       "      <td>83965</td>\n",
       "      <td>160008</td>\n",
       "      <td>288365</td>\n",
       "      <td>346486</td>\n",
       "      <td>334643</td>\n",
       "      <td>12387</td>\n",
       "      <td>278900</td>\n",
       "      <td>27442</td>\n",
       "      <td>311438</td>\n",
       "      <td>175989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.7058</td>\n",
       "      <td>-0.1327</td>\n",
       "      <td>5.1025</td>\n",
       "      <td>-14.3858</td>\n",
       "      <td>17.8630</td>\n",
       "      <td>23.2274</td>\n",
       "      <td>1.4375</td>\n",
       "      <td>14.4838</td>\n",
       "      <td>...</td>\n",
       "      <td>193746</td>\n",
       "      <td>55131</td>\n",
       "      <td>355825</td>\n",
       "      <td>374237</td>\n",
       "      <td>91741</td>\n",
       "      <td>44310</td>\n",
       "      <td>272654</td>\n",
       "      <td>17443</td>\n",
       "      <td>29860</td>\n",
       "      <td>365488</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 402 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code  target    var_0    var_1  var_10  var_100  var_101  var_102  \\\n",
       "0  test_0     NaN  11.0656   7.7798 -2.0248  -9.2198  17.3089  30.9548   \n",
       "1  test_1     NaN   8.5304   1.2543 -1.3809  -1.7257  15.4712  35.6020   \n",
       "2  test_2     NaN   5.4827 -10.3581 -4.7057  -3.5065  14.1663  28.0256   \n",
       "3  test_3     NaN   8.5374  -1.3222  0.0095   1.7021   2.5363   3.8763   \n",
       "4  test_4     NaN  11.7058  -0.1327  5.1025 -14.3858  17.8630  23.2274   \n",
       "\n",
       "   var_103  var_104     ...       var_90_rank  var_91_rank  var_92_rank  \\\n",
       "0   1.4918  12.8721     ...            283352         7357       136022   \n",
       "1   1.6570  13.0783     ...             57818       384042       254703   \n",
       "2   1.3935  10.8257     ...             79010        34644       232915   \n",
       "3   1.5173  13.4083     ...             83965       160008       288365   \n",
       "4   1.4375  14.4838     ...            193746        55131       355825   \n",
       "\n",
       "   var_93_rank  var_94_rank  var_95_rank  var_96_rank  var_97_rank  \\\n",
       "0       153965       202685       380958       384304        14213   \n",
       "1       226045       324501        41135        14516       332780   \n",
       "2       316034       380224       127703       379528       307586   \n",
       "3       346486       334643        12387       278900        27442   \n",
       "4       374237        91741        44310       272654        17443   \n",
       "\n",
       "   var_98_rank  var_99_rank  \n",
       "0       247572       368780  \n",
       "1        96953       387184  \n",
       "2       137103       347787  \n",
       "3       311438       175989  \n",
       "4        29860       365488  \n",
       "\n",
       "[5 rows x 402 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(rank_models, 'rank_aug_models.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_answer(models, answer_qdf):\n",
    "    test_labels = answer_qdf.ID_code\n",
    "    answer_qdf = answer_qdf.drop(['ID_code','target'], axis=1)\n",
    "    y_preds = {}\n",
    "    for i, model in enumerate(models):\n",
    "            print(f\"On fold: {i}\")\n",
    "            y_preds[str(i)] = model.predict(answer_qdf)\n",
    "    y_preds = pd.DataFrame(y_preds)\n",
    "    answer_df = pd.DataFrame({\n",
    "        'ID_code' : test_labels,\n",
    "        'target' : y_preds.mean(axis = 1),\n",
    "        })\n",
    "    return answer_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On fold: 0\n",
      "On fold: 1\n",
      "On fold: 2\n",
      "On fold: 3\n",
      "On fold: 4\n",
      "On fold: 5\n",
      "On fold: 6\n",
      "On fold: 7\n",
      "On fold: 8\n",
      "On fold: 9\n"
     ]
    }
   ],
   "source": [
    "answer = make_answer(rank_models, test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.157681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.266446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.248564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.283618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.060358</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    target\n",
       "0  test_0  0.157681\n",
       "1  test_1  0.266446\n",
       "2  test_2  0.248564\n",
       "3  test_3  0.283618\n",
       "4  test_4  0.060358"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "answer_10_gpu_lgbm.csv\r\n",
      "answer_11_gpu_lgbm_and_pred_best.csv\r\n",
      "answer_12_lgbm_bin_trainednotonval.csv\r\n",
      "answer_13_lgbm_bin_trainednotonval.csv\r\n",
      "answer_14_my_scores_blended.csv\r\n",
      "answer_15_probability_no_aug.csv\r\n",
      "answer_16_probability_lolThiswontWork.csv\r\n",
      "answer_17_kendall_correlation_inverse_weighted_blend.csv\r\n",
      "answer_18_blend_average.csv\r\n",
      "answer_19_kendall_correlation_inverse_weighted_blend_with_larger_difference.csv\r\n",
      "answer_1_simple_gbm.csv\r\n",
      "answer_2_lgbm_with_CV10.csv\r\n",
      "answer_3_lgbm_with_augmented_data_smote_and_shiz.csv\r\n",
      "answer_4_lgbm_with_augmented_data_smote_and_shiz_and_super_learner.csv\r\n",
      "answer_5_lgbm_with_only_random_augmented_data.csv\r\n",
      "answer_6_lgbm_with_best_and_random_averaged.csv\r\n",
      "answer_7_lgbm_ORbwRandomAndBest.csv\r\n",
      "answer_8_lgbm_AveragedORbwRandomAndBest.csv\r\n",
      "answer_9_lb_aug.csv\r\n",
      "init3.csv\r\n",
      "paugTest.csv\r\n",
      "paugTrain.csv\r\n",
      "test_augmented.csv\r\n",
      "train_augmented.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls *.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer.to_csv('answer_20_lgbm_rank_augmented.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " adasyn_aug_lgbm_models_cv10.m\r\n",
      " answer_10_gpu_lgbm.csv\r\n",
      " answer_11_gpu_lgbm_and_pred_best.csv\r\n",
      " answer_12_lgbm_bin_trainednotonval.csv\r\n",
      " answer_13_lgbm_bin_trainednotonval.csv\r\n",
      " answer_14_my_scores_blended.csv\r\n",
      " answer_15_probability_no_aug.csv\r\n",
      " answer_16_probability_lolThiswontWork.csv\r\n",
      " answer_17_kendall_correlation_inverse_weighted_blend.csv\r\n",
      " answer_18_blend_average.csv\r\n",
      " answer_19_kendall_correlation_inverse_weighted_blend_with_larger_difference.csv\r\n",
      " answer_1_simple_gbm.csv\r\n",
      " answer_20_lgbm_rank_augmented.csv\r\n",
      " answer_2_lgbm_with_CV10.csv\r\n",
      " answer_3_lgbm_with_augmented_data_smote_and_shiz.csv\r\n",
      " answer_4_lgbm_with_augmented_data_smote_and_shiz_and_super_learner.csv\r\n",
      " answer_5_lgbm_with_only_random_augmented_data.csv\r\n",
      " answer_6_lgbm_with_best_and_random_averaged.csv\r\n",
      " answer_7_lgbm_ORbwRandomAndBest.csv\r\n",
      " answer_8_lgbm_AveragedORbwRandomAndBest.csv\r\n",
      " answer_9_lb_aug.csv\r\n",
      "'argsort feature augment.ipynb'\r\n",
      "'Augmented Data - Answer generate.ipynb'\r\n",
      "'Augmented LightGBM model .ipynb'\r\n",
      "'Bayesian tuning - lgbm and xgboost.ipynb'\r\n",
      " bsmote_aug_lgbm_models_cv10.m\r\n",
      " data\r\n",
      "'Data Agumentation.ipynb'\r\n",
      " fastai\r\n",
      "'gotta get past 901.ipynb'\r\n",
      " helpers.py\r\n",
      " idk_stupid_bin.m\r\n",
      " init3.csv\r\n",
      "'install gpu lightgbm.ipynb'\r\n",
      " latest_lgb_models.m\r\n",
      " latest_xgb_models.m\r\n",
      " lb_aug_901_cv10.m\r\n",
      "'LB augment cheated [latest].ipynb'\r\n",
      " lgbm_cv10_models.m\r\n",
      " LightGBM\r\n",
      "'Outlier detection - All categorical - .ipynb'\r\n",
      " paugTest.csv\r\n",
      " paugTrain.csv\r\n",
      "'Porbability features.ipynb'\r\n",
      " probs_model_augmented_on_val.m\r\n",
      " probs_model_no_augmented.m\r\n",
      " __pycache__\r\n",
      " random_aug_lgbm_models_cv10.m\r\n",
      " rank_aug_models.m\r\n",
      " rf_lgbm_cv10.ipynb\r\n",
      " Santander_v1.ipynb\r\n",
      " smote_aug_lgbm_models_cv10.m\r\n",
      " stacking\r\n",
      " Stacking.ipynb\r\n",
      " test_augmented.csv\r\n",
      " test_profile.html\r\n",
      " todo.txt\r\n",
      " train_augmented.csv\r\n",
      " train_profile.html\r\n",
      " Untitled1.ipynb\r\n",
      " Untitled.ipynb\r\n",
      " visuals.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.139434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.270568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.235634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.296368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.060134</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    target\n",
       "0  test_0  0.139434\n",
       "1  test_1  0.270568\n",
       "2  test_2  0.235634\n",
       "3  test_3  0.296368\n",
       "4  test_4  0.060134"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = pd.read_csv('stacking/2019-03-20_15_43_sub.csv')\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}